[ Aug 12 22:32:17 Enabled. ]
[ Aug 12 22:32:17 Rereading configuration. ]
[ Aug 12 22:32:20 Rereading configuration. ]
[ Aug 12 22:32:26 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_537dde5b-089b-4e20-9ef8-bf10127d7dce/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_537dde5b-089b-4e20-9ef8-bf10127d7dce/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:32:26.812456597Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:32:26.815706906Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:32:26.815716736Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:32:26.826257756Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:32:26.985531972Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.513666917Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:41774"}
{"msg":"region b754dae4-8dfb-4055-91cb-eacd8945042e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.527913281Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.53085567Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.534186741Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"uri":"/crucible/0/regions","method":"POST","req_id":"8f1828b0-2ebe-4192-a914-4952a2631b8f","remote_addr":"[fd00:1122:3344:101::a]:41774","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":11210,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_537dde5b-089b-4e20-9ef8-bf10127d7dce/crucible/regions/b754dae4-8dfb-4055-91cb-eacd8945042e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.541310068Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_537dde5b-089b-4e20-9ef8-bf10127d7dce/crucible/regions/b754dae4-8dfb-4055-91cb-eacd8945042e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.541331069Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b754dae4-8dfb-4055-91cb-eacd8945042e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b754dae4-8dfb-4055-91cb-eacd8945042e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.600257881Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"region":"b754dae4-8dfb-4055-91cb-eacd8945042e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b754dae4-8dfb-4055-91cb-eacd8945042e\" \"--data\" \"/data/regions/b754dae4-8dfb-4055-91cb-eacd8945042e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.600341034Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"region":"b754dae4-8dfb-4055-91cb-eacd8945042e","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.834886409Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"region":"b754dae4-8dfb-4055-91cb-eacd8945042e","component":"worker"}
{"msg":"region b754dae4-8dfb-4055-91cb-eacd8945042e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.83494469Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.835019583Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b754dae4-8dfb-4055-91cb-eacd8945042e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.835458016Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b754dae4-8dfb-4055-91cb-eacd8945042e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.841247742Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.858456256Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b754dae4-8dfb-4055-91cb-eacd8945042e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.862473658Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b754dae4-8dfb-4055-91cb-eacd8945042e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.869734009Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.87174789Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.871814052Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.871864153Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.877445643Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.880113155Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"uri":"/crucible/0/regions","method":"POST","req_id":"72994bc1-c282-4cd5-8aee-57d34b6d7fe6","remote_addr":"[fd00:1122:3344:101::a]:41774","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:33:38.884585351Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.117055752Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:58077"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.117162495Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"uri":"/crucible/0/regions/b754dae4-8dfb-4055-91cb-eacd8945042e","method":"GET","req_id":"ed37f809-92e5-40d9-8275-2d7af25e49ea","remote_addr":"[fd00:1122:3344:101::a]:58077","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"path is \"/data/regions/b754dae4-8dfb-4055-91cb-eacd8945042e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.123198409Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"datafile"}
{"msg":"dataset is oxp_537dde5b-089b-4e20-9ef8-bf10127d7dce/crucible/regions/b754dae4-8dfb-4055-91cb-eacd8945042e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.133382609Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"datafile"}
{"msg":"snapshot is oxp_537dde5b-089b-4e20-9ef8-bf10127d7dce/crucible/regions/b754dae4-8dfb-4055-91cb-eacd8945042e@6ad175c4-d009-4baf-bcf4-40a5a6345162","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.141290259Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_537dde5b-089b-4e20-9ef8-bf10127d7dce/crucible/regions/b754dae4-8dfb-4055-91cb-eacd8945042e\", \"6ad175c4-d009-4baf-bcf4-40a5a6345162\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.14132441Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.152520751Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"uri":"/crucible/0/regions/b754dae4-8dfb-4055-91cb-eacd8945042e/snapshots/6ad175c4-d009-4baf-bcf4-40a5a6345162","method":"GET","req_id":"76205fba-011d-4bc5-a82d-7bc254b23b14","remote_addr":"[fd00:1122:3344:101::a]:58077","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":32451,"response_code":"200"}
{"msg":"path is \"/data/regions/b754dae4-8dfb-4055-91cb-eacd8945042e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.155410999Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"datafile"}
{"msg":"dataset is oxp_537dde5b-089b-4e20-9ef8-bf10127d7dce/crucible/regions/b754dae4-8dfb-4055-91cb-eacd8945042e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.163417482Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"datafile"}
{"msg":"snapshot is oxp_537dde5b-089b-4e20-9ef8-bf10127d7dce/crucible/regions/b754dae4-8dfb-4055-91cb-eacd8945042e@6ad175c4-d009-4baf-bcf4-40a5a6345162","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.172194539Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_537dde5b-089b-4e20-9ef8-bf10127d7dce/crucible/regions/b754dae4-8dfb-4055-91cb-eacd8945042e\", \"6ad175c4-d009-4baf-bcf4-40a5a6345162\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.17222939Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot b754dae4-8dfb-4055-91cb-eacd8945042e-6ad175c4-d009-4baf-bcf4-40a5a6345162 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.181645047Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"datafile"}
{"msg":"applying SMF actions for region b754dae4-8dfb-4055-91cb-eacd8945042e running snapshot 6ad175c4-d009-4baf-bcf4-40a5a6345162 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.182191553Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.182238245Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"uri":"/crucible/0/regions/b754dae4-8dfb-4055-91cb-eacd8945042e/snapshots/6ad175c4-d009-4baf-bcf4-40a5a6345162/run","method":"POST","req_id":"71539f92-5d6c-47c6-a9ac-6cdfcbbaef16","remote_addr":"[fd00:1122:3344:101::a]:58077","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":26856,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/b754dae4-8dfb-4055-91cb-eacd8945042e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.1876832Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.187765553Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.187845745Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.187853415Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.187898077Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b754dae4-8dfb-4055-91cb-eacd8945042e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.187947178Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b754dae4-8dfb-4055-91cb-eacd8945042e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.188044531Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.188106693Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.188153215Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.188200346Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.199910972Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-b754dae4-8dfb-4055-91cb-eacd8945042e-6ad175c4-d009-4baf-bcf4-40a5a6345162","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.200439518Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-b754dae4-8dfb-4055-91cb-eacd8945042e-6ad175c4-d009-4baf-bcf4-40a5a6345162","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.210106162Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-b754dae4-8dfb-4055-91cb-eacd8945042e-6ad175c4-d009-4baf-bcf4-40a5a6345162","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.230693599Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b754dae4-8dfb-4055-91cb-eacd8945042e/.zfs/snapshot/6ad175c4-d009-4baf-bcf4-40a5a6345162","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.230989998Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.231044729Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.231085971Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.231125752Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.231179744Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.238048822Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.247727667Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"region b754dae4-8dfb-4055-91cb-eacd8945042e running snapshot 6ad175c4-d009-4baf-bcf4-40a5a6345162 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.247745097Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"datafile"}
{"msg":"path is \"/data/regions/b754dae4-8dfb-4055-91cb-eacd8945042e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.47007142Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"datafile"}
{"msg":"dataset is oxp_537dde5b-089b-4e20-9ef8-bf10127d7dce/crucible/regions/b754dae4-8dfb-4055-91cb-eacd8945042e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.479218188Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"datafile"}
{"msg":"snapshot is oxp_537dde5b-089b-4e20-9ef8-bf10127d7dce/crucible/regions/b754dae4-8dfb-4055-91cb-eacd8945042e@6ad175c4-d009-4baf-bcf4-40a5a6345162","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.487638974Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_537dde5b-089b-4e20-9ef8-bf10127d7dce/crucible/regions/b754dae4-8dfb-4055-91cb-eacd8945042e\", \"6ad175c4-d009-4baf-bcf4-40a5a6345162\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.487667575Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:15.496484973Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"uri":"/crucible/0/regions/b754dae4-8dfb-4055-91cb-eacd8945042e/snapshots/6ad175c4-d009-4baf-bcf4-40a5a6345162/run","method":"POST","req_id":"4014b7da-12bc-4473-867b-d0e3b0e5455d","remote_addr":"[fd00:1122:3344:101::a]:58077","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":26449,"response_code":"200"}
{"msg":"region e86b1522-e9d2-4e32-9e6c-f281bda6ce88 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.71701204Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.717173685Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"uri":"/crucible/0/regions","method":"POST","req_id":"026b6c00-e4ee-4070-ae46-34e30acbdb2d","remote_addr":"[fd00:1122:3344:101::a]:58077","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":855,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.717203606Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_537dde5b-089b-4e20-9ef8-bf10127d7dce/crucible/regions/e86b1522-e9d2-4e32-9e6c-f281bda6ce88","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.726105016Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_537dde5b-089b-4e20-9ef8-bf10127d7dce/crucible/regions/e86b1522-e9d2-4e32-9e6c-f281bda6ce88","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.726133157Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e86b1522-e9d2-4e32-9e6c-f281bda6ce88\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e86b1522-e9d2-4e32-9e6c-f281bda6ce88\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.811112852Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"region":"e86b1522-e9d2-4e32-9e6c-f281bda6ce88","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e86b1522-e9d2-4e32-9e6c-f281bda6ce88\" \"--data\" \"/data/regions/e86b1522-e9d2-4e32-9e6c-f281bda6ce88\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.812979589Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"region":"e86b1522-e9d2-4e32-9e6c-f281bda6ce88","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.956628809Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"region":"e86b1522-e9d2-4e32-9e6c-f281bda6ce88","component":"worker"}
{"msg":"region e86b1522-e9d2-4e32-9e6c-f281bda6ce88 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.956682831Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.956785824Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/b754dae4-8dfb-4055-91cb-eacd8945042e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.957393902Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.957474395Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.957556277Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.957563307Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.957606929Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b754dae4-8dfb-4055-91cb-eacd8945042e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.95765839Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b754dae4-8dfb-4055-91cb-eacd8945042e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.958042022Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.958107904Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.958164796Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.958212957Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.965517199Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e86b1522-e9d2-4e32-9e6c-f281bda6ce88","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.965983643Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e86b1522-e9d2-4e32-9e6c-f281bda6ce88","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.972245424Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.988464517Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e86b1522-e9d2-4e32-9e6c-f281bda6ce88","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.995939605Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e86b1522-e9d2-4e32-9e6c-f281bda6ce88","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.996159732Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.996217543Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.996261424Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:18.996301916Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:19.001129513Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:19.011165238Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:19.064885792Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"uri":"/crucible/0/regions","method":"POST","req_id":"95c422c2-b664-4686-90dc-bf26a73e16b6","remote_addr":"[fd00:1122:3344:101::a]:58077","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.353043763Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"uri":"/crucible/0/regions/e86b1522-e9d2-4e32-9e6c-f281bda6ce88","method":"GET","req_id":"31f7f629-fa77-480d-b03d-54e81cce8d43","remote_addr":"[fd00:1122:3344:101::a]:58077","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":46,"response_code":"200"}
{"msg":"path is \"/data/regions/e86b1522-e9d2-4e32-9e6c-f281bda6ce88\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.355538739Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"datafile"}
{"msg":"dataset is oxp_537dde5b-089b-4e20-9ef8-bf10127d7dce/crucible/regions/e86b1522-e9d2-4e32-9e6c-f281bda6ce88","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.367957037Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"datafile"}
{"msg":"region e86b1522-e9d2-4e32-9e6c-f281bda6ce88 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.376371453Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.376573829Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.376637311Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"uri":"/crucible/0/regions/e86b1522-e9d2-4e32-9e6c-f281bda6ce88","method":"DELETE","req_id":"0fea5a49-a7f3-41cd-a8ad-201124c899e5","remote_addr":"[fd00:1122:3344:101::a]:58077","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":21131,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-e86b1522-e9d2-4e32-9e6c-f281bda6ce88 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.377663603Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.380059175Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"uri":"/crucible/0/regions/e86b1522-e9d2-4e32-9e6c-f281bda6ce88","method":"GET","req_id":"5b706446-54a3-4a20-b698-263ed78620a0","remote_addr":"[fd00:1122:3344:101::a]:58077","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":450,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/b754dae4-8dfb-4055-91cb-eacd8945042e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.382015055Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.382099947Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.38218377Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.38219548Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.382234731Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b754dae4-8dfb-4055-91cb-eacd8945042e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.382284963Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b754dae4-8dfb-4055-91cb-eacd8945042e","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.382589422Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.382653534Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.382716686Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.382778938Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.386825131Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.388245754Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_537dde5b-089b-4e20-9ef8-bf10127d7dce/crucible/regions/e86b1522-e9d2-4e32-9e6c-f281bda6ce88\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.398151576Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"region":"e86b1522-e9d2-4e32-9e6c-f281bda6ce88","component":"worker"}
{"msg":"region e86b1522-e9d2-4e32-9e6c-f281bda6ce88 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.471101354Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-12T22:39:44.476846189Z","hostname":"oxz_crucible_b5dc5628-b92d-454b-bdaf-72d2635fbb46","pid":22541,"uri":"/crucible/0/regions/e86b1522-e9d2-4e32-9e6c-f281bda6ce88","method":"GET","req_id":"3627bd68-0d6b-466b-b246-cc2681ec4334","remote_addr":"[fd00:1122:3344:101::a]:58077","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":85,"response_code":"200"}