[ Sep  6 23:36:38 Enabled. ]
[ Sep  6 23:36:38 Rereading configuration. ]
[ Sep  6 23:36:41 Rereading configuration. ]
[ Sep  6 23:36:43 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ab9a060c-a3d5-48de-ba57-beaa10d18520/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_ab9a060c-a3d5-48de-ba57-beaa10d18520/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:36:43.375105766Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:36:43.379007189Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:36:43.379025189Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:36:43.402843042Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:36:43.532075464Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.722521036Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:38779"}
{"msg":"region f826107b-a5dc-4ec9-9656-6ef226e74767 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.762509289Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.767684359Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"uri":"/crucible/0/regions","method":"POST","req_id":"b749921d-1e6d-472c-82f9-1d32c846f6c4","remote_addr":"[fd00:1122:3344:101::a]:38779","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":24793,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.772784998Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_ab9a060c-a3d5-48de-ba57-beaa10d18520/crucible/regions/f826107b-a5dc-4ec9-9656-6ef226e74767","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.782943656Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ab9a060c-a3d5-48de-ba57-beaa10d18520/crucible/regions/f826107b-a5dc-4ec9-9656-6ef226e74767","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.782976926Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f826107b-a5dc-4ec9-9656-6ef226e74767\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f826107b-a5dc-4ec9-9656-6ef226e74767\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.841142845Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"region":"f826107b-a5dc-4ec9-9656-6ef226e74767","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f826107b-a5dc-4ec9-9656-6ef226e74767\" \"--data\" \"/data/regions/f826107b-a5dc-4ec9-9656-6ef226e74767\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.843651549Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"region":"f826107b-a5dc-4ec9-9656-6ef226e74767","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:15.097451609Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"uri":"/crucible/0/regions","method":"POST","req_id":"804bf3ea-eca4-4f14-b0f3-dfbabc3e6ed3","remote_addr":"[fd00:1122:3344:101::a]:38779","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:15.150274446Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"region":"f826107b-a5dc-4ec9-9656-6ef226e74767","component":"worker"}
{"msg":"region f826107b-a5dc-4ec9-9656-6ef226e74767 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:15.150303966Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:15.150458518Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f826107b-a5dc-4ec9-9656-6ef226e74767","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:15.150855403Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f826107b-a5dc-4ec9-9656-6ef226e74767","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:15.155813941Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:15.171928059Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f826107b-a5dc-4ec9-9656-6ef226e74767","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:15.175323855Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f826107b-a5dc-4ec9-9656-6ef226e74767","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:15.184949636Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:15.18751208Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:15.187581121Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:15.187621622Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:15.192051522Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:15.199101828Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:15.475919261Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"uri":"/crucible/0/regions","method":"POST","req_id":"9ca06757-6dd6-4414-b561-44702ee0915a","remote_addr":"[fd00:1122:3344:101::a]:38779","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.670029334Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45374"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.67266489Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"uri":"/crucible/0/regions/f826107b-a5dc-4ec9-9656-6ef226e74767","method":"GET","req_id":"9659c3b7-be72-42a7-a126-0abc6c58c76a","remote_addr":"[fd00:1122:3344:101::a]:45374","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":2603,"response_code":"200"}
{"msg":"path is \"/data/regions/f826107b-a5dc-4ec9-9656-6ef226e74767\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.675321206Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"datafile"}
{"msg":"dataset is oxp_ab9a060c-a3d5-48de-ba57-beaa10d18520/crucible/regions/f826107b-a5dc-4ec9-9656-6ef226e74767","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.685009177Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"datafile"}
{"msg":"snapshot is oxp_ab9a060c-a3d5-48de-ba57-beaa10d18520/crucible/regions/f826107b-a5dc-4ec9-9656-6ef226e74767@14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.694335634Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ab9a060c-a3d5-48de-ba57-beaa10d18520/crucible/regions/f826107b-a5dc-4ec9-9656-6ef226e74767\", \"14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.694362824Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.705088729Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"uri":"/crucible/0/regions/f826107b-a5dc-4ec9-9656-6ef226e74767/snapshots/14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed","method":"GET","req_id":"41bb7f49-320c-454d-8517-5284946d5555","remote_addr":"[fd00:1122:3344:101::a]:45374","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":29786,"response_code":"200"}
{"msg":"path is \"/data/regions/f826107b-a5dc-4ec9-9656-6ef226e74767\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.707922018Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"datafile"}
{"msg":"dataset is oxp_ab9a060c-a3d5-48de-ba57-beaa10d18520/crucible/regions/f826107b-a5dc-4ec9-9656-6ef226e74767","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.7162132Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"datafile"}
{"msg":"snapshot is oxp_ab9a060c-a3d5-48de-ba57-beaa10d18520/crucible/regions/f826107b-a5dc-4ec9-9656-6ef226e74767@14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.724442852Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ab9a060c-a3d5-48de-ba57-beaa10d18520/crucible/regions/f826107b-a5dc-4ec9-9656-6ef226e74767\", \"14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.724471952Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot f826107b-a5dc-4ec9-9656-6ef226e74767-14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.733536885Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"datafile"}
{"msg":"applying SMF actions for region f826107b-a5dc-4ec9-9656-6ef226e74767 running snapshot 14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.734471698Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.734501978Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"uri":"/crucible/0/regions/f826107b-a5dc-4ec9-9656-6ef226e74767/snapshots/14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed/run","method":"POST","req_id":"03dc8419-6cfb-4d7b-adfb-b2fe83602aea","remote_addr":"[fd00:1122:3344:101::a]:45374","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":26613,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/f826107b-a5dc-4ec9-9656-6ef226e74767","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.743064524Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.743166446Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.743265567Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.743273267Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.743321168Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f826107b-a5dc-4ec9-9656-6ef226e74767","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.743372559Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f826107b-a5dc-4ec9-9656-6ef226e74767","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.743663472Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.746208527Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.746272378Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.746333509Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.755631185Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-f826107b-a5dc-4ec9-9656-6ef226e74767-14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.756099651Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-f826107b-a5dc-4ec9-9656-6ef226e74767-14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.762411267Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-f826107b-a5dc-4ec9-9656-6ef226e74767-14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.779183704Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f826107b-a5dc-4ec9-9656-6ef226e74767/.zfs/snapshot/14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.779488268Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.77956789Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.77961921Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.779682931Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.779749372Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.78473418Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.792083759Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"region f826107b-a5dc-4ec9-9656-6ef226e74767 running snapshot 14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:37.792102329Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"datafile"}
{"msg":"path is \"/data/regions/f826107b-a5dc-4ec9-9656-6ef226e74767\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.086679903Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"datafile"}
{"msg":"dataset is oxp_ab9a060c-a3d5-48de-ba57-beaa10d18520/crucible/regions/f826107b-a5dc-4ec9-9656-6ef226e74767","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.095768336Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"datafile"}
{"msg":"snapshot is oxp_ab9a060c-a3d5-48de-ba57-beaa10d18520/crucible/regions/f826107b-a5dc-4ec9-9656-6ef226e74767@14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.103539061Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ab9a060c-a3d5-48de-ba57-beaa10d18520/crucible/regions/f826107b-a5dc-4ec9-9656-6ef226e74767\", \"14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.103584532Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.112996399Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"uri":"/crucible/0/regions/f826107b-a5dc-4ec9-9656-6ef226e74767/snapshots/14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed/run","method":"POST","req_id":"bdb01926-5553-4f55-912c-4e6faf6f5908","remote_addr":"[fd00:1122:3344:101::a]:45374","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":26348,"response_code":"200"}
{"msg":"region f6ee60d9-84fc-40a6-863c-0787247df677 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:41.980763501Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:41.980956474Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"uri":"/crucible/0/regions","method":"POST","req_id":"387d74ac-0f3d-4f4e-bcfa-9ca1d4afecb0","remote_addr":"[fd00:1122:3344:101::a]:45374","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":264,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:41.980986554Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_ab9a060c-a3d5-48de-ba57-beaa10d18520/crucible/regions/f6ee60d9-84fc-40a6-863c-0787247df677","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:41.990545654Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ab9a060c-a3d5-48de-ba57-beaa10d18520/crucible/regions/f6ee60d9-84fc-40a6-863c-0787247df677","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:41.990574634Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f6ee60d9-84fc-40a6-863c-0787247df677\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f6ee60d9-84fc-40a6-863c-0787247df677\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.071000915Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"region":"f6ee60d9-84fc-40a6-863c-0787247df677","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f6ee60d9-84fc-40a6-863c-0787247df677\" \"--data\" \"/data/regions/f6ee60d9-84fc-40a6-863c-0787247df677\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.07285077Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"region":"f6ee60d9-84fc-40a6-863c-0787247df677","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.19459464Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"region":"f6ee60d9-84fc-40a6-863c-0787247df677","component":"worker"}
{"msg":"region f6ee60d9-84fc-40a6-863c-0787247df677 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.194633581Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.194746123Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f6ee60d9-84fc-40a6-863c-0787247df677","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.195159248Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f6ee60d9-84fc-40a6-863c-0787247df677","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.202117553Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.216166893Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f6ee60d9-84fc-40a6-863c-0787247df677","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.222349807Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f6ee60d9-84fc-40a6-863c-0787247df677","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.22259549Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.222668201Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.222730092Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.222788783Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.230368256Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/f826107b-a5dc-4ec9-9656-6ef226e74767","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.240424292Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.240530274Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.240644595Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.240653745Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.240707736Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f826107b-a5dc-4ec9-9656-6ef226e74767","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.240768527Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f826107b-a5dc-4ec9-9656-6ef226e74767","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.248662424Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.248720025Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.248770175Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.248819046Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.25499389Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.257119678Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:42.375416502Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"uri":"/crucible/0/regions","method":"POST","req_id":"76859d96-696e-46cc-8688-6e3ee30ae369","remote_addr":"[fd00:1122:3344:101::a]:45374","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":485,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.074704902Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"uri":"/crucible/0/regions/f6ee60d9-84fc-40a6-863c-0787247df677","method":"GET","req_id":"07e98f34-c279-47ef-910a-2706e757c5cd","remote_addr":"[fd00:1122:3344:101::a]:45374","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"path is \"/data/regions/f6ee60d9-84fc-40a6-863c-0787247df677\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.080733423Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"datafile"}
{"msg":"dataset is oxp_ab9a060c-a3d5-48de-ba57-beaa10d18520/crucible/regions/f6ee60d9-84fc-40a6-863c-0787247df677","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.092434322Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"datafile"}
{"msg":"region f6ee60d9-84fc-40a6-863c-0787247df677 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.100052485Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.100333059Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.10040835Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"uri":"/crucible/0/regions/f6ee60d9-84fc-40a6-863c-0787247df677","method":"DELETE","req_id":"02eaa57d-8b72-43df-8901-a5b0036f36ef","remote_addr":"[fd00:1122:3344:101::a]:45374","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":20288,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-f6ee60d9-84fc-40a6-863c-0787247df677 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.101511505Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.103303389Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"uri":"/crucible/0/regions/f6ee60d9-84fc-40a6-863c-0787247df677","method":"GET","req_id":"c6d4c930-c33c-4e50-ac25-45c164e6fcc0","remote_addr":"[fd00:1122:3344:101::a]:45374","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":23,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/f826107b-a5dc-4ec9-9656-6ef226e74767","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.108329277Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.108420989Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.10850563Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.10851896Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.108556411Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f826107b-a5dc-4ec9-9656-6ef226e74767","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.108603991Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f826107b-a5dc-4ec9-9656-6ef226e74767","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.108956666Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.109020387Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.109071048Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.109118468Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.112352112Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.113784311Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ab9a060c-a3d5-48de-ba57-beaa10d18520/crucible/regions/f6ee60d9-84fc-40a6-863c-0787247df677\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.123546284Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"region":"f6ee60d9-84fc-40a6-863c-0787247df677","component":"worker"}
{"msg":"region f6ee60d9-84fc-40a6-863c-0787247df677 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.170623042Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:44:25.255843148Z","hostname":"oxz_crucible_0a5539c1-7e26-4e50-b26a-60a137388ce1","pid":19307,"uri":"/crucible/0/regions/f6ee60d9-84fc-40a6-863c-0787247df677","method":"GET","req_id":"547d9b4b-d3da-464d-9281-8dbd77ae3103","remote_addr":"[fd00:1122:3344:101::a]:45374","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":62,"response_code":"200"}