[ Oct  9 23:15:59 Enabled. ]
[ Oct  9 23:15:59 Rereading configuration. ]
[ Oct  9 23:16:01 Rereading configuration. ]
[ Oct  9 23:16:04 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ef6af0ca-2803-42da-991f-fb732a7b3eda/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_ef6af0ca-2803-42da-991f-fb732a7b3eda/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:04.973936674Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:04.975544955Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:04.975552716Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:04.992778386Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:05.197293821Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:19.833802396Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:50338"}
{"msg":"region c1171a08-abc1-45bf-b887-f2fc70943458 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:19.861239889Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:19.867147265Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"uri":"/crucible/0/regions","method":"POST","req_id":"a358b959-4019-4a12-8986-4fcad46318f7","remote_addr":"[fd00:1122:3344:101::a]:50338","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":14010,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:19.868314179Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_ef6af0ca-2803-42da-991f-fb732a7b3eda/crucible/regions/c1171a08-abc1-45bf-b887-f2fc70943458","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:19.877113533Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ef6af0ca-2803-42da-991f-fb732a7b3eda/crucible/regions/c1171a08-abc1-45bf-b887-f2fc70943458","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:19.877141683Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c1171a08-abc1-45bf-b887-f2fc70943458\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c1171a08-abc1-45bf-b887-f2fc70943458\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:19.928348166Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"region":"c1171a08-abc1-45bf-b887-f2fc70943458","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c1171a08-abc1-45bf-b887-f2fc70943458\" \"--data\" \"/data/regions/c1171a08-abc1-45bf-b887-f2fc70943458\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:19.932778623Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"region":"c1171a08-abc1-45bf-b887-f2fc70943458","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:20.070780613Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"region":"c1171a08-abc1-45bf-b887-f2fc70943458","component":"worker"}
{"msg":"region c1171a08-abc1-45bf-b887-f2fc70943458 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:20.070834094Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:20.070909095Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c1171a08-abc1-45bf-b887-f2fc70943458","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:20.07315665Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c1171a08-abc1-45bf-b887-f2fc70943458","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:20.077739701Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:20.096249317Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c1171a08-abc1-45bf-b887-f2fc70943458","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:20.100135904Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c1171a08-abc1-45bf-b887-f2fc70943458","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:20.10753616Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:20.107598711Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:20.107650592Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:20.107688603Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:20.111206843Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:20.116605599Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:17:20.164207161Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"uri":"/crucible/0/regions","method":"POST","req_id":"ccd251ea-43b5-4751-aeb9-1981e866f2bd","remote_addr":"[fd00:1122:3344:101::a]:50338","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":66,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.466340498Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:35736"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.474722754Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"uri":"/crucible/0/regions/c1171a08-abc1-45bf-b887-f2fc70943458","method":"GET","req_id":"f56601da-a15e-4751-b384-2ea31c648a86","remote_addr":"[fd00:1122:3344:101::a]:35736","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":8267,"response_code":"200"}
{"msg":"path is \"/data/regions/c1171a08-abc1-45bf-b887-f2fc70943458\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.478283567Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"datafile"}
{"msg":"dataset is oxp_ef6af0ca-2803-42da-991f-fb732a7b3eda/crucible/regions/c1171a08-abc1-45bf-b887-f2fc70943458","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.487232982Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"datafile"}
{"msg":"snapshot is oxp_ef6af0ca-2803-42da-991f-fb732a7b3eda/crucible/regions/c1171a08-abc1-45bf-b887-f2fc70943458@c171cea4-eebc-44fb-83e6-14ac51e1d1ff","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.495638018Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ef6af0ca-2803-42da-991f-fb732a7b3eda/crucible/regions/c1171a08-abc1-45bf-b887-f2fc70943458\", \"c171cea4-eebc-44fb-83e6-14ac51e1d1ff\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.495682469Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.516144763Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"uri":"/crucible/0/regions/c1171a08-abc1-45bf-b887-f2fc70943458/snapshots/c171cea4-eebc-44fb-83e6-14ac51e1d1ff","method":"GET","req_id":"7f7f1e4d-84bb-4540-a2d1-ef2b2d68b119","remote_addr":"[fd00:1122:3344:101::a]:35736","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":37872,"response_code":"200"}
{"msg":"path is \"/data/regions/c1171a08-abc1-45bf-b887-f2fc70943458\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.522902487Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"datafile"}
{"msg":"dataset is oxp_ef6af0ca-2803-42da-991f-fb732a7b3eda/crucible/regions/c1171a08-abc1-45bf-b887-f2fc70943458","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.53062784Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"datafile"}
{"msg":"snapshot is oxp_ef6af0ca-2803-42da-991f-fb732a7b3eda/crucible/regions/c1171a08-abc1-45bf-b887-f2fc70943458@c171cea4-eebc-44fb-83e6-14ac51e1d1ff","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.53822295Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ef6af0ca-2803-42da-991f-fb732a7b3eda/crucible/regions/c1171a08-abc1-45bf-b887-f2fc70943458\", \"c171cea4-eebc-44fb-83e6-14ac51e1d1ff\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.538272651Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot c1171a08-abc1-45bf-b887-f2fc70943458-c171cea4-eebc-44fb-83e6-14ac51e1d1ff state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.547110446Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.54781811Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"uri":"/crucible/0/regions/c1171a08-abc1-45bf-b887-f2fc70943458/snapshots/c171cea4-eebc-44fb-83e6-14ac51e1d1ff/run","method":"POST","req_id":"dbc27167-813c-4fc3-ad70-29e361f5a994","remote_addr":"[fd00:1122:3344:101::a]:35736","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":25503,"response_code":"200"}
{"msg":"applying SMF actions for region c1171a08-abc1-45bf-b887-f2fc70943458 running snapshot c171cea4-eebc-44fb-83e6-14ac51e1d1ff (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.54784441Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/c1171a08-abc1-45bf-b887-f2fc70943458","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.562693444Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.562859087Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.56295922Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.56297214Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.56300263Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c1171a08-abc1-45bf-b887-f2fc70943458","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.563064802Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c1171a08-abc1-45bf-b887-f2fc70943458","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.563165724Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.563221735Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.563269435Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.563329327Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.575365594Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-c1171a08-abc1-45bf-b887-f2fc70943458-c171cea4-eebc-44fb-83e6-14ac51e1d1ff","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.575925556Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-c1171a08-abc1-45bf-b887-f2fc70943458-c171cea4-eebc-44fb-83e6-14ac51e1d1ff","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.588678568Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-c1171a08-abc1-45bf-b887-f2fc70943458-c171cea4-eebc-44fb-83e6-14ac51e1d1ff","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.634910472Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c1171a08-abc1-45bf-b887-f2fc70943458/.zfs/snapshot/c171cea4-eebc-44fb-83e6-14ac51e1d1ff","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.635208728Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.635266859Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.63530973Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.635348761Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.635396292Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.664867225Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.675506705Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"region c1171a08-abc1-45bf-b887-f2fc70943458 running snapshot c171cea4-eebc-44fb-83e6-14ac51e1d1ff state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.675544936Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"datafile"}
{"msg":"path is \"/data/regions/c1171a08-abc1-45bf-b887-f2fc70943458\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.795354616Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"datafile"}
{"msg":"dataset is oxp_ef6af0ca-2803-42da-991f-fb732a7b3eda/crucible/regions/c1171a08-abc1-45bf-b887-f2fc70943458","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.803793352Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"datafile"}
{"msg":"snapshot is oxp_ef6af0ca-2803-42da-991f-fb732a7b3eda/crucible/regions/c1171a08-abc1-45bf-b887-f2fc70943458@c171cea4-eebc-44fb-83e6-14ac51e1d1ff","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.812054846Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ef6af0ca-2803-42da-991f-fb732a7b3eda/crucible/regions/c1171a08-abc1-45bf-b887-f2fc70943458\", \"c171cea4-eebc-44fb-83e6-14ac51e1d1ff\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.812084256Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:42.820656686Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"uri":"/crucible/0/regions/c1171a08-abc1-45bf-b887-f2fc70943458/snapshots/c171cea4-eebc-44fb-83e6-14ac51e1d1ff/run","method":"POST","req_id":"7599b1ec-8f6a-43f8-b8aa-3c2fc12a2756","remote_addr":"[fd00:1122:3344:101::a]:35736","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":25305,"response_code":"200"}
{"msg":"region db8e06fe-289c-4e42-955e-f220a16e6cc9 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.452174307Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.452344251Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"uri":"/crucible/0/regions","method":"POST","req_id":"fe0fc80f-240f-43d0-8121-3c24ef28f887","remote_addr":"[fd00:1122:3344:101::a]:35736","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":245,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.452374521Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_ef6af0ca-2803-42da-991f-fb732a7b3eda/crucible/regions/db8e06fe-289c-4e42-955e-f220a16e6cc9","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.46191954Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ef6af0ca-2803-42da-991f-fb732a7b3eda/crucible/regions/db8e06fe-289c-4e42-955e-f220a16e6cc9","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.461949191Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"db8e06fe-289c-4e42-955e-f220a16e6cc9\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/db8e06fe-289c-4e42-955e-f220a16e6cc9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.551854289Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"region":"db8e06fe-289c-4e42-955e-f220a16e6cc9","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"db8e06fe-289c-4e42-955e-f220a16e6cc9\" \"--data\" \"/data/regions/db8e06fe-289c-4e42-955e-f220a16e6cc9\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.553706465Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"region":"db8e06fe-289c-4e42-955e-f220a16e6cc9","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.598659624Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"uri":"/crucible/0/regions","method":"POST","req_id":"6460af96-9fd5-4354-9758-450372bede60","remote_addr":"[fd00:1122:3344:101::a]:35736","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.751079269Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"region":"db8e06fe-289c-4e42-955e-f220a16e6cc9","component":"worker"}
{"msg":"region db8e06fe-289c-4e42-955e-f220a16e6cc9 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.75111555Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.751241222Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/c1171a08-abc1-45bf-b887-f2fc70943458","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.751842914Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.751926846Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.752007527Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.752020857Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.752055598Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c1171a08-abc1-45bf-b887-f2fc70943458","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.75212041Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c1171a08-abc1-45bf-b887-f2fc70943458","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.752483537Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.752545178Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.752593459Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.75264454Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.780867478Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-db8e06fe-289c-4e42-955e-f220a16e6cc9","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.781370808Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-db8e06fe-289c-4e42-955e-f220a16e6cc9","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.788820696Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.811545745Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-db8e06fe-289c-4e42-955e-f220a16e6cc9","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.81837471Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/db8e06fe-289c-4e42-955e-f220a16e6cc9","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.818661956Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.818731267Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.818782368Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.818834909Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.830096442Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:47.83860336Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:20:48.339819563Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"uri":"/crucible/0/regions","method":"POST","req_id":"f76aecfc-344d-43b4-8d25-2c5daf1c3dce","remote_addr":"[fd00:1122:3344:101::a]:35736","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":61,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.429046093Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"uri":"/crucible/0/regions/db8e06fe-289c-4e42-955e-f220a16e6cc9","method":"GET","req_id":"0e866775-4ed1-43c3-8776-1b28ad945a77","remote_addr":"[fd00:1122:3344:101::a]:35736","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":51,"response_code":"200"}
{"msg":"path is \"/data/regions/db8e06fe-289c-4e42-955e-f220a16e6cc9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.433677795Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"datafile"}
{"msg":"dataset is oxp_ef6af0ca-2803-42da-991f-fb732a7b3eda/crucible/regions/db8e06fe-289c-4e42-955e-f220a16e6cc9","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.447157461Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"datafile"}
{"msg":"region db8e06fe-289c-4e42-955e-f220a16e6cc9 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.457525836Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.457743691Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-db8e06fe-289c-4e42-955e-f220a16e6cc9 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.458860863Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.46024402Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"uri":"/crucible/0/regions/db8e06fe-289c-4e42-955e-f220a16e6cc9","method":"DELETE","req_id":"800ba679-d36d-4564-8bbd-ecfd9921c365","remote_addr":"[fd00:1122:3344:101::a]:35736","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":27335,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.463624317Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"uri":"/crucible/0/regions/db8e06fe-289c-4e42-955e-f220a16e6cc9","method":"GET","req_id":"b64815e3-f787-48e8-be06-ed5270903a69","remote_addr":"[fd00:1122:3344:101::a]:35736","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":23,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/c1171a08-abc1-45bf-b887-f2fc70943458","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.470412131Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.470520343Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.470644736Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.470659916Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.470711487Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c1171a08-abc1-45bf-b887-f2fc70943458","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.470769598Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c1171a08-abc1-45bf-b887-f2fc70943458","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.471297779Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.47138681Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.471457672Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.471518983Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.480396279Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.482096112Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ef6af0ca-2803-42da-991f-fb732a7b3eda/crucible/regions/db8e06fe-289c-4e42-955e-f220a16e6cc9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.492705792Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"region":"db8e06fe-289c-4e42-955e-f220a16e6cc9","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.554395872Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"uri":"/crucible/0/regions/db8e06fe-289c-4e42-955e-f220a16e6cc9","method":"GET","req_id":"9e1a8872-8c31-46f4-9306-06fe135684e8","remote_addr":"[fd00:1122:3344:101::a]:35736","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"region db8e06fe-289c-4e42-955e-f220a16e6cc9 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.566358248Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:21:14.729415673Z","hostname":"oxz_crucible_074083cb-db23-46c8-8ec1-8074ad72183f","pid":24453,"uri":"/crucible/0/regions/db8e06fe-289c-4e42-955e-f220a16e6cc9","method":"GET","req_id":"b80de951-576b-49bf-a666-4f8aa7f6d28e","remote_addr":"[fd00:1122:3344:101::a]:35736","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":36,"response_code":"200"}