[ Mar 12 16:15:54 Enabled. ]
[ Mar 12 16:15:54 Rereading configuration. ]
[ Mar 12 16:15:56 Rereading configuration. ]
[ Mar 12 16:16:01 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_e06bdc7e-b17a-4859-b4ab-79eac4284485/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:01.825390821Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370}
{"msg":"dataset: \"oxp_e06bdc7e-b17a-4859-b4ab-79eac4284485/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:01.825603525Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:01.825648805Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:01.825655986Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:01.839113919Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:01.942041832Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.400431796Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:37493"}
{"msg":"region f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.427487006Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.430225208Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.430281359Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"uri":"/crucible/0/regions","method":"POST","req_id":"93e65600-06b1-432e-b8a3-5519df001826","remote_addr":"[fd00:1122:3344:101::b]:37493","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":14539,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_e06bdc7e-b17a-4859-b4ab-79eac4284485/crucible/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.439393221Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_e06bdc7e-b17a-4859-b4ab-79eac4284485/crucible/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.439415171Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.506369095Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"region":"f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af\" \"--data\" \"/data/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.508694969Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"region":"f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.628664282Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"uri":"/crucible/0/regions","method":"POST","req_id":"f68bc843-2018-4a92-a82f-494c370dfd27","remote_addr":"[fd00:1122:3344:101::b]:37493","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":63,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.876649132Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"region":"f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af","component":"worker"}
{"msg":"region f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.876731183Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.876908847Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.877245743Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.882664825Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.900506512Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.903977308Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.904689241Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.904770743Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.904814723Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.904847084Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.909353209Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:49.914303692Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:16:50.180997975Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"uri":"/crucible/0/regions","method":"POST","req_id":"68a00e68-a61e-4959-b528-0a3b199fbecf","remote_addr":"[fd00:1122:3344:101::b]:37493","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":61,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.326923093Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:60914"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.327656377Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"uri":"/crucible/0/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af","method":"GET","req_id":"c77db6c4-4c81-4e32-b165-58650be747a3","remote_addr":"[fd00:1122:3344:101::b]:60914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":82,"response_code":200}
{"msg":"path is \"/data/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.332793674Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"datafile"}
{"msg":"dataset is oxp_e06bdc7e-b17a-4859-b4ab-79eac4284485/crucible/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.343249681Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"datafile"}
{"msg":"snapshot is oxp_e06bdc7e-b17a-4859-b4ab-79eac4284485/crucible/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af@93db9137-4507-4fbd-9812-efd7964088c5","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.352880273Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_e06bdc7e-b17a-4859-b4ab-79eac4284485/crucible/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af\", \"93db9137-4507-4fbd-9812-efd7964088c5\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.352928454Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.365935859Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"uri":"/crucible/0/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af/snapshots/93db9137-4507-4fbd-9812-efd7964088c5","method":"GET","req_id":"c213c6eb-db32-4233-bcec-850545fc660d","remote_addr":"[fd00:1122:3344:101::b]:60914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":33175,"response_code":200}
{"msg":"path is \"/data/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.370163349Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"datafile"}
{"msg":"dataset is oxp_e06bdc7e-b17a-4859-b4ab-79eac4284485/crucible/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.377689201Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"datafile"}
{"msg":"snapshot is oxp_e06bdc7e-b17a-4859-b4ab-79eac4284485/crucible/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af@93db9137-4507-4fbd-9812-efd7964088c5","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.38559011Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_e06bdc7e-b17a-4859-b4ab-79eac4284485/crucible/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af\", \"93db9137-4507-4fbd-9812-efd7964088c5\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.385629331Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af-93db9137-4507-4fbd-9812-efd7964088c5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.396301542Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.396945294Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"uri":"/crucible/0/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af/snapshots/93db9137-4507-4fbd-9812-efd7964088c5/run","method":"POST","req_id":"b529fc78-c485-4b44-8834-8d3d4549d9e8","remote_addr":"[fd00:1122:3344:101::b]:60914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":26813,"response_code":200}
{"msg":"applying SMF actions for region f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af running snapshot 93db9137-4507-4fbd-9812-efd7964088c5 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.396969545Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af-93db9137-4507-4fbd-9812-efd7964088c5","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.427911499Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af-93db9137-4507-4fbd-9812-efd7964088c5","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.436181695Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af-93db9137-4507-4fbd-9812-efd7964088c5","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.451412852Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af/.zfs/snapshot/93db9137-4507-4fbd-9812-efd7964088c5","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.451782409Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.451853841Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.451910231Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.451965353Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.452038724Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.456797014Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.462560003Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"region f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af running snapshot 93db9137-4507-4fbd-9812-efd7964088c5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.462573463Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"datafile"}
{"msg":"path is \"/data/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.537050448Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"datafile"}
{"msg":"dataset is oxp_e06bdc7e-b17a-4859-b4ab-79eac4284485/crucible/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.543271055Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"datafile"}
{"msg":"snapshot is oxp_e06bdc7e-b17a-4859-b4ab-79eac4284485/crucible/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af@93db9137-4507-4fbd-9812-efd7964088c5","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.549741348Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_e06bdc7e-b17a-4859-b4ab-79eac4284485/crucible/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af\", \"93db9137-4507-4fbd-9812-efd7964088c5\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.549788409Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:55.555315173Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"uri":"/crucible/0/regions/f7ca51e9-6ebf-4a7d-bf85-00c79c4a05af/snapshots/93db9137-4507-4fbd-9812-efd7964088c5/run","method":"POST","req_id":"43512596-5aad-4d48-bd81-cad67d38890f","remote_addr":"[fd00:1122:3344:101::b]:60914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":18322,"response_code":200}
{"msg":"region 6627b7e6-3382-4897-adcd-f7ae263cbf25 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.797639424Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.797808047Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"uri":"/crucible/0/regions","method":"POST","req_id":"5e0556f9-ea7c-4ab8-bbbb-f9d6cc5a2cef","remote_addr":"[fd00:1122:3344:101::b]:60914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":215,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.797834277Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_e06bdc7e-b17a-4859-b4ab-79eac4284485/crucible/regions/6627b7e6-3382-4897-adcd-f7ae263cbf25","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.804070885Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_e06bdc7e-b17a-4859-b4ab-79eac4284485/crucible/regions/6627b7e6-3382-4897-adcd-f7ae263cbf25","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.804100956Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6627b7e6-3382-4897-adcd-f7ae263cbf25\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6627b7e6-3382-4897-adcd-f7ae263cbf25\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.853458057Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"region":"6627b7e6-3382-4897-adcd-f7ae263cbf25","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6627b7e6-3382-4897-adcd-f7ae263cbf25\" \"--data\" \"/data/regions/6627b7e6-3382-4897-adcd-f7ae263cbf25\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.853502118Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"region":"6627b7e6-3382-4897-adcd-f7ae263cbf25","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.937939031Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"uri":"/crucible/0/regions","method":"POST","req_id":"ddd5eb76-0573-48d3-85d3-86baba2313ef","remote_addr":"[fd00:1122:3344:101::b]:60914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":50,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.944481755Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"region":"6627b7e6-3382-4897-adcd-f7ae263cbf25","component":"worker"}
{"msg":"region 6627b7e6-3382-4897-adcd-f7ae263cbf25 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.944513935Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.944621297Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6627b7e6-3382-4897-adcd-f7ae263cbf25","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.944984904Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6627b7e6-3382-4897-adcd-f7ae263cbf25","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.958171793Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.965360949Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6627b7e6-3382-4897-adcd-f7ae263cbf25","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.968735193Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6627b7e6-3382-4897-adcd-f7ae263cbf25","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.968998937Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.969048908Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.969088439Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.96912672Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.972618736Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:27:59.982477142Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:28:00.436989338Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"uri":"/crucible/0/regions","method":"POST","req_id":"2fbf1281-1ed0-4535-bc0e-74de41e1828b","remote_addr":"[fd00:1122:3344:101::b]:60914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":62,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:28:24.862465454Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"uri":"/crucible/0/regions/6627b7e6-3382-4897-adcd-f7ae263cbf25","method":"GET","req_id":"a0550ce8-d0fc-4761-92da-ab405fa64c61","remote_addr":"[fd00:1122:3344:101::b]:60914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":57,"response_code":200}
{"msg":"path is \"/data/regions/6627b7e6-3382-4897-adcd-f7ae263cbf25\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:28:24.865723905Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"datafile"}
{"msg":"dataset is oxp_e06bdc7e-b17a-4859-b4ab-79eac4284485/crucible/regions/6627b7e6-3382-4897-adcd-f7ae263cbf25","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:28:24.877258583Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"datafile"}
{"msg":"region 6627b7e6-3382-4897-adcd-f7ae263cbf25 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:28:24.884146523Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:28:24.884759294Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"uri":"/crucible/0/regions/6627b7e6-3382-4897-adcd-f7ae263cbf25","method":"DELETE","req_id":"469fcf25-c16b-4a90-b9ad-e4fc68a4411a","remote_addr":"[fd00:1122:3344:101::b]:60914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":19089,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:28:24.884791205Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-6627b7e6-3382-4897-adcd-f7ae263cbf25 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:28:24.886384225Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:28:24.888009146Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"uri":"/crucible/0/regions/6627b7e6-3382-4897-adcd-f7ae263cbf25","method":"GET","req_id":"67ad1053-5d6c-4b06-9345-4ab38f6d7892","remote_addr":"[fd00:1122:3344:101::b]:60914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":32,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:28:24.901060532Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_e06bdc7e-b17a-4859-b4ab-79eac4284485/crucible/regions/6627b7e6-3382-4897-adcd-f7ae263cbf25\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:28:24.908701306Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"region":"6627b7e6-3382-4897-adcd-f7ae263cbf25","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:28:24.950948683Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"uri":"/crucible/0/regions/6627b7e6-3382-4897-adcd-f7ae263cbf25","method":"GET","req_id":"729f3eb6-3aa1-4065-bdbd-aac8209f8c67","remote_addr":"[fd00:1122:3344:101::b]:60914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":36,"response_code":200}
{"msg":"region 6627b7e6-3382-4897-adcd-f7ae263cbf25 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:28:24.955812935Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T16:28:25.133877315Z","hostname":"oxz_crucible_3f31292c-597d-44f0-bad4-5b0c3bb780d2","pid":28370,"uri":"/crucible/0/regions/6627b7e6-3382-4897-adcd-f7ae263cbf25","method":"GET","req_id":"13a06610-708b-4926-8b6a-f2bcb6a740e6","remote_addr":"[fd00:1122:3344:101::b]:60914","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":56,"response_code":200}