[ Sep  3 23:51:13 Enabled. ]
[ Sep  3 23:51:13 Rereading configuration. ]
[ Sep  3 23:51:14 Rereading configuration. ]
[ Sep  3 23:51:16 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_93f19386-05be-4b48-8c7f-97b089021599/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_93f19386-05be-4b48-8c7f-97b089021599/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:16.494149935Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:16.498629452Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:16.498641932Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:16.519064322Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:16.60165841Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.460361677Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64380"}
{"msg":"region 41721d83-a3d8-4524-99f4-71e64871f37e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.494098343Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.499287865Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"uri":"/crucible/0/regions","method":"POST","req_id":"1801ec0e-993e-4c50-9a2c-f26aa49a81f8","remote_addr":"[fd00:1122:3344:101::a]:64380","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":22488,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.50046301Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_93f19386-05be-4b48-8c7f-97b089021599/crucible/regions/41721d83-a3d8-4524-99f4-71e64871f37e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.513772017Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_93f19386-05be-4b48-8c7f-97b089021599/crucible/regions/41721d83-a3d8-4524-99f4-71e64871f37e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.513837788Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"41721d83-a3d8-4524-99f4-71e64871f37e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/41721d83-a3d8-4524-99f4-71e64871f37e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.601429485Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"region":"41721d83-a3d8-4524-99f4-71e64871f37e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"41721d83-a3d8-4524-99f4-71e64871f37e\" \"--data\" \"/data/regions/41721d83-a3d8-4524-99f4-71e64871f37e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.603374507Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"region":"41721d83-a3d8-4524-99f4-71e64871f37e","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.739219382Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"region":"41721d83-a3d8-4524-99f4-71e64871f37e","component":"worker"}
{"msg":"region 41721d83-a3d8-4524-99f4-71e64871f37e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.739310504Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.739513228Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-41721d83-a3d8-4524-99f4-71e64871f37e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.739957088Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-41721d83-a3d8-4524-99f4-71e64871f37e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.743243059Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.757622868Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-41721d83-a3d8-4524-99f4-71e64871f37e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.762404721Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/41721d83-a3d8-4524-99f4-71e64871f37e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.771109919Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.773113942Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.773174993Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.773230825Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.777296462Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.784132659Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:51:50.842442504Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"uri":"/crucible/0/regions","method":"POST","req_id":"2517ffe5-d00a-4965-884d-c4df78320a14","remote_addr":"[fd00:1122:3344:101::a]:64380","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":58,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.683539532Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45518"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.686572187Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"uri":"/crucible/0/regions/41721d83-a3d8-4524-99f4-71e64871f37e","method":"GET","req_id":"5ccbfaa5-3b22-4830-bad1-08650c7af202","remote_addr":"[fd00:1122:3344:101::a]:45518","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":3013,"response_code":"200"}
{"msg":"path is \"/data/regions/41721d83-a3d8-4524-99f4-71e64871f37e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.689345897Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"datafile"}
{"msg":"dataset is oxp_93f19386-05be-4b48-8c7f-97b089021599/crucible/regions/41721d83-a3d8-4524-99f4-71e64871f37e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.697908081Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"datafile"}
{"msg":"snapshot is oxp_93f19386-05be-4b48-8c7f-97b089021599/crucible/regions/41721d83-a3d8-4524-99f4-71e64871f37e@6cb8a04e-0263-457c-acc7-970602f6584b","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.706246821Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_93f19386-05be-4b48-8c7f-97b089021599/crucible/regions/41721d83-a3d8-4524-99f4-71e64871f37e\", \"6cb8a04e-0263-457c-acc7-970602f6584b\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.706276961Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.71692477Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"uri":"/crucible/0/regions/41721d83-a3d8-4524-99f4-71e64871f37e/snapshots/6cb8a04e-0263-457c-acc7-970602f6584b","method":"GET","req_id":"088324d7-fa34-4173-921b-205b4bc8e3e9","remote_addr":"[fd00:1122:3344:101::a]:45518","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":27595,"response_code":"200"}
{"msg":"path is \"/data/regions/41721d83-a3d8-4524-99f4-71e64871f37e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.720969507Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"datafile"}
{"msg":"dataset is oxp_93f19386-05be-4b48-8c7f-97b089021599/crucible/regions/41721d83-a3d8-4524-99f4-71e64871f37e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.729231905Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"datafile"}
{"msg":"snapshot is oxp_93f19386-05be-4b48-8c7f-97b089021599/crucible/regions/41721d83-a3d8-4524-99f4-71e64871f37e@6cb8a04e-0263-457c-acc7-970602f6584b","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.737710468Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_93f19386-05be-4b48-8c7f-97b089021599/crucible/regions/41721d83-a3d8-4524-99f4-71e64871f37e\", \"6cb8a04e-0263-457c-acc7-970602f6584b\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.737740779Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 41721d83-a3d8-4524-99f4-71e64871f37e-6cb8a04e-0263-457c-acc7-970602f6584b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.746419575Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"datafile"}
{"msg":"applying SMF actions for region 41721d83-a3d8-4524-99f4-71e64871f37e running snapshot 6cb8a04e-0263-457c-acc7-970602f6584b (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.747029039Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.747055669Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"uri":"/crucible/0/regions/41721d83-a3d8-4524-99f4-71e64871f37e/snapshots/6cb8a04e-0263-457c-acc7-970602f6584b/run","method":"POST","req_id":"fd68ec26-1465-4f86-ba93-57cfb9d1ab9e","remote_addr":"[fd00:1122:3344:101::a]:45518","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":26546,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/41721d83-a3d8-4524-99f4-71e64871f37e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.757857102Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.757962614Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.758062476Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.758070737Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.758121258Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-41721d83-a3d8-4524-99f4-71e64871f37e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.758184949Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/41721d83-a3d8-4524-99f4-71e64871f37e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.758294641Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.761265655Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.761319556Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.761370297Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.768193235Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-41721d83-a3d8-4524-99f4-71e64871f37e-6cb8a04e-0263-457c-acc7-970602f6584b","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.768660425Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-41721d83-a3d8-4524-99f4-71e64871f37e-6cb8a04e-0263-457c-acc7-970602f6584b","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.771535716Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-41721d83-a3d8-4524-99f4-71e64871f37e-6cb8a04e-0263-457c-acc7-970602f6584b","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.782218126Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/41721d83-a3d8-4524-99f4-71e64871f37e/.zfs/snapshot/6cb8a04e-0263-457c-acc7-970602f6584b","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.782520593Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.782573094Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.782615285Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.782652616Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.782698017Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.786466618Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.790867343Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"region 41721d83-a3d8-4524-99f4-71e64871f37e running snapshot 6cb8a04e-0263-457c-acc7-970602f6584b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:57.790886353Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"datafile"}
{"msg":"path is \"/data/regions/41721d83-a3d8-4524-99f4-71e64871f37e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:58.007257742Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"datafile"}
{"msg":"dataset is oxp_93f19386-05be-4b48-8c7f-97b089021599/crucible/regions/41721d83-a3d8-4524-99f4-71e64871f37e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:58.016155294Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"datafile"}
{"msg":"snapshot is oxp_93f19386-05be-4b48-8c7f-97b089021599/crucible/regions/41721d83-a3d8-4524-99f4-71e64871f37e@6cb8a04e-0263-457c-acc7-970602f6584b","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:58.024992884Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_93f19386-05be-4b48-8c7f-97b089021599/crucible/regions/41721d83-a3d8-4524-99f4-71e64871f37e\", \"6cb8a04e-0263-457c-acc7-970602f6584b\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:58.025034145Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:56:58.03360985Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"uri":"/crucible/0/regions/41721d83-a3d8-4524-99f4-71e64871f37e/snapshots/6cb8a04e-0263-457c-acc7-970602f6584b/run","method":"POST","req_id":"c2406670-55eb-4f93-a2fe-afd6269f63c5","remote_addr":"[fd00:1122:3344:101::a]:45518","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":26397,"response_code":"200"}
{"msg":"region eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:01.98489747Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:01.985057974Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"uri":"/crucible/0/regions","method":"POST","req_id":"a9987266-6596-4865-8e48-5da8c8cbac83","remote_addr":"[fd00:1122:3344:101::a]:45518","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":224,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:01.985094945Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_93f19386-05be-4b48-8c7f-97b089021599/crucible/regions/eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:01.99507406Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_93f19386-05be-4b48-8c7f-97b089021599/crucible/regions/eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:01.99510431Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.05686067Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"region":"eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f\" \"--data\" \"/data/regions/eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.056919121Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"region":"eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.140562293Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"region":"eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f","component":"worker"}
{"msg":"region eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.140643274Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.140723736Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/41721d83-a3d8-4524-99f4-71e64871f37e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.141300798Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.14138247Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.141462342Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.141468902Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.141511293Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-41721d83-a3d8-4524-99f4-71e64871f37e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.141561184Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/41721d83-a3d8-4524-99f4-71e64871f37e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.141945592Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.142010144Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.142060675Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.142108206Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.145696863Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.146117712Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.152108591Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.159284596Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.162486905Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.16274299Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.162810672Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.162855663Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.162896003Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.167007482Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.172786396Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:02.267435775Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"uri":"/crucible/0/regions","method":"POST","req_id":"b6a3032e-fc13-4bc2-aef9-2228b0a990f6","remote_addr":"[fd00:1122:3344:101::a]:45518","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":706,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.420040413Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"uri":"/crucible/0/regions/eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f","method":"GET","req_id":"11c7ac40-1002-478f-9877-ee503cb972c2","remote_addr":"[fd00:1122:3344:101::a]:45518","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":50,"response_code":"200"}
{"msg":"path is \"/data/regions/eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.423503247Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"datafile"}
{"msg":"dataset is oxp_93f19386-05be-4b48-8c7f-97b089021599/crucible/regions/eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.433584644Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"datafile"}
{"msg":"region eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.441143347Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.441356502Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.441413093Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"uri":"/crucible/0/regions/eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f","method":"DELETE","req_id":"fce04dd7-9c91-41c1-988b-edfcb64dd109","remote_addr":"[fd00:1122:3344:101::a]:45518","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":17945,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.442422845Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.444285185Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"uri":"/crucible/0/regions/eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f","method":"GET","req_id":"a2560761-6043-4a3b-8835-d4416cb71bce","remote_addr":"[fd00:1122:3344:101::a]:45518","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":21,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/41721d83-a3d8-4524-99f4-71e64871f37e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.44636813Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.446474242Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.446560864Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.446571344Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.446626575Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-41721d83-a3d8-4524-99f4-71e64871f37e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.446681746Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/41721d83-a3d8-4524-99f4-71e64871f37e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.447038204Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.447096675Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.447145047Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.447191688Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.450923318Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.452314798Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_93f19386-05be-4b48-8c7f-97b089021599/crucible/regions/eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.461520126Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"region":"eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f","component":"worker"}
{"msg":"region eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.528750764Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T23:57:27.595158934Z","hostname":"oxz_crucible_69c63c12-bcf3-4c66-841c-9be9bd48eb10","pid":22359,"uri":"/crucible/0/regions/eb1ddb58-dab9-4ee4-91fe-e1e7aa45cc4f","method":"GET","req_id":"14ef2610-8e0b-4c50-9ed8-fce0490cd5cc","remote_addr":"[fd00:1122:3344:101::a]:45518","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":61,"response_code":"200"}