[ Mar 10 10:15:56 Enabled. ]
[ Mar 10 10:15:56 Rereading configuration. ]
[ Mar 10 10:15:57 Rereading configuration. ]
[ Mar 10 10:16:01 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:16:01.182966623Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210}
{"msg":"dataset: \"oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:16:01.183868823Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:16:01.183881273Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:16:01.183888294Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:16:01.19838919Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:16:01.283251279Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.344263815Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53910"}
{"msg":"region db188a23-b696-48a1-8676-b8f8ca3c4ea5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.36935484Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.371512828Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.371530589Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"uri":"/crucible/0/regions","method":"POST","req_id":"c163d089-6f11-4a49-bf15-33a05b4d1866","remote_addr":"[fd00:1122:3344:101::a]:53910","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":12997,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.379418776Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.379437667Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"db188a23-b696-48a1-8676-b8f8ca3c4ea5\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.422632788Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"region":"db188a23-b696-48a1-8676-b8f8ca3c4ea5","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"db188a23-b696-48a1-8676-b8f8ca3c4ea5\" \"--data\" \"/data/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.425134395Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"region":"db188a23-b696-48a1-8676-b8f8ca3c4ea5","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.649681127Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"uri":"/crucible/0/regions","method":"POST","req_id":"4fa4c572-2d02-4f0f-b2ac-8bc7c49b6ba8","remote_addr":"[fd00:1122:3344:101::a]:53910","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":60,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.773741689Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"region":"db188a23-b696-48a1-8676-b8f8ca3c4ea5","component":"worker"}
{"msg":"region db188a23-b696-48a1-8676-b8f8ca3c4ea5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.7737871Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.773883552Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-db188a23-b696-48a1-8676-b8f8ca3c4ea5","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.77425878Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-db188a23-b696-48a1-8676-b8f8ca3c4ea5","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.77956312Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.794542467Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-db188a23-b696-48a1-8676-b8f8ca3c4ea5","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.797415941Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.798105037Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.798174429Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.798217559Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.7982535Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.80178699Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.806298541Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:17:04.920848059Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"uri":"/crucible/0/regions","method":"POST","req_id":"b0e29af5-ee72-4d6e-82cb-22f5252cc9b1","remote_addr":"[fd00:1122:3344:101::a]:53910","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":59,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.637565171Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:37930"}
{"msg":"region 9560c5d8-6009-4b65-a45f-7400182cfc96 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.638206546Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.638591314Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"uri":"/crucible/0/regions","method":"POST","req_id":"4c9e6847-2e13-452c-9663-361ac361fcf6","remote_addr":"[fd00:1122:3344:101::a]:37930","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":928,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.638610155Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible/regions/9560c5d8-6009-4b65-a45f-7400182cfc96","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.647158537Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible/regions/9560c5d8-6009-4b65-a45f-7400182cfc96","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.647186337Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"9560c5d8-6009-4b65-a45f-7400182cfc96\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/9560c5d8-6009-4b65-a45f-7400182cfc96\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.703707289Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"region":"9560c5d8-6009-4b65-a45f-7400182cfc96","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"9560c5d8-6009-4b65-a45f-7400182cfc96\" \"--data\" \"/data/regions/9560c5d8-6009-4b65-a45f-7400182cfc96\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.70374885Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"region":"9560c5d8-6009-4b65-a45f-7400182cfc96","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.787892944Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"uri":"/crucible/0/regions","method":"POST","req_id":"d3befba7-153d-4df1-8f72-321f67db40b8","remote_addr":"[fd00:1122:3344:101::a]:37930","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":56,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.858565254Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"region":"9560c5d8-6009-4b65-a45f-7400182cfc96","component":"worker"}
{"msg":"region 9560c5d8-6009-4b65-a45f-7400182cfc96 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.858596115Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.858739698Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-9560c5d8-6009-4b65-a45f-7400182cfc96","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.859151697Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-9560c5d8-6009-4b65-a45f-7400182cfc96","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.871491145Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.900255482Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9560c5d8-6009-4b65-a45f-7400182cfc96","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.906254097Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9560c5d8-6009-4b65-a45f-7400182cfc96","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.906521983Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.906565984Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.906614475Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.906650686Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.912341454Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:12.935287841Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:13.276186912Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"uri":"/crucible/0/regions","method":"POST","req_id":"d7371010-a682-4dcf-8254-a49e4cd85ad8","remote_addr":"[fd00:1122:3344:101::a]:37930","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":60,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.671344656Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"uri":"/crucible/0/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5","method":"GET","req_id":"08d08d1c-906f-478a-9aab-552a0a21661b","remote_addr":"[fd00:1122:3344:101::a]:37930","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":82,"response_code":200}
{"msg":"path is \"/data/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.675651643Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"dataset is oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.682979678Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"snapshot is oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5@c0faf504-0c1c-4d68-aec2-791428cd2be6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.689161607Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5\", \"c0faf504-0c1c-4d68-aec2-791428cd2be6\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.689180498Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.696453611Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"uri":"/crucible/0/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5/snapshots/c0faf504-0c1c-4d68-aec2-791428cd2be6","method":"GET","req_id":"b496523b-9c27-4f40-a4d9-574e9542aa4d","remote_addr":"[fd00:1122:3344:101::a]:37930","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":20823,"response_code":200}
{"msg":"path is \"/data/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.700045612Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"dataset is oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.705391632Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"snapshot is oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5@c0faf504-0c1c-4d68-aec2-791428cd2be6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.710701112Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5\", \"c0faf504-0c1c-4d68-aec2-791428cd2be6\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.710718712Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot db188a23-b696-48a1-8676-b8f8ca3c4ea5-c0faf504-0c1c-4d68-aec2-791428cd2be6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.719728395Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.719863358Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"uri":"/crucible/0/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5/snapshots/c0faf504-0c1c-4d68-aec2-791428cd2be6/run","method":"POST","req_id":"4baf4b58-4c4b-4ec8-b7a4-39e4b20b6b4c","remote_addr":"[fd00:1122:3344:101::a]:37930","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":19847,"response_code":200}
{"msg":"applying SMF actions for region db188a23-b696-48a1-8676-b8f8ca3c4ea5 running snapshot c0faf504-0c1c-4d68-aec2-791428cd2be6 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.719896849Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-db188a23-b696-48a1-8676-b8f8ca3c4ea5-c0faf504-0c1c-4d68-aec2-791428cd2be6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.724353949Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-db188a23-b696-48a1-8676-b8f8ca3c4ea5-c0faf504-0c1c-4d68-aec2-791428cd2be6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.739647973Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-db188a23-b696-48a1-8676-b8f8ca3c4ea5-c0faf504-0c1c-4d68-aec2-791428cd2be6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.755573562Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5/.zfs/snapshot/c0faf504-0c1c-4d68-aec2-791428cd2be6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.755910039Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.755968661Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.756017222Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.756065123Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.756136454Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.761183758Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.767589762Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"region db188a23-b696-48a1-8676-b8f8ca3c4ea5 running snapshot c0faf504-0c1c-4d68-aec2-791428cd2be6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.767608412Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"path is \"/data/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.865977046Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"dataset is oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.873174818Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"snapshot is oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5@c0faf504-0c1c-4d68-aec2-791428cd2be6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.880319299Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5\", \"c0faf504-0c1c-4d68-aec2-791428cd2be6\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.880341599Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:14.887172193Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"uri":"/crucible/0/regions/db188a23-b696-48a1-8676-b8f8ca3c4ea5/snapshots/c0faf504-0c1c-4d68-aec2-791428cd2be6/run","method":"POST","req_id":"ff569329-4ae5-4c82-b0ba-9edbef118a53","remote_addr":"[fd00:1122:3344:101::a]:37930","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":21245,"response_code":200}
{"msg":"region f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.457833474Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.458001998Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"uri":"/crucible/0/regions","method":"POST","req_id":"cc30de58-906e-476c-abc1-53c35d072b69","remote_addr":"[fd00:1122:3344:101::a]:37930","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":220,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.458021878Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible/regions/f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.464906053Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible/regions/f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.464925793Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.528673467Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"region":"f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86\" \"--data\" \"/data/regions/f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.528715368Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"region":"f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.629483495Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"region":"f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86","component":"worker"}
{"msg":"region f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.629507356Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.629627109Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.631302856Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.634783835Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.651434429Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.656269578Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.656581065Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.656622146Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.656655587Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.656692718Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.665284071Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.6763573Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:20.809287772Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"uri":"/crucible/0/regions","method":"POST","req_id":"ddd5133d-302d-40af-9826-937aa31606b6","remote_addr":"[fd00:1122:3344:101::a]:37930","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":75,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:51.178277824Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:38039"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:51.178482069Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"uri":"/crucible/0/regions/f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86","method":"GET","req_id":"5ced3b10-7ec5-46a3-983e-4ffb6f9d9f04","remote_addr":"[fd00:1122:3344:101::a]:38039","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":48,"response_code":200}
{"msg":"path is \"/data/regions/f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:51.180944664Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"dataset is oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible/regions/f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:51.191372949Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"region f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:51.199035721Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:51.199612134Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:51.199622094Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"uri":"/crucible/0/regions/f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86","method":"DELETE","req_id":"d63141cf-33a1-4fd7-907f-bf06789c6e82","remote_addr":"[fd00:1122:3344:101::a]:38039","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":18689,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:51.200856912Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:51.204570076Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"uri":"/crucible/0/regions/f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86","method":"GET","req_id":"90cb0ded-40f6-4fca-b5f7-b794d4b2be4c","remote_addr":"[fd00:1122:3344:101::a]:38039","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":43,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:51.217261001Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_16701fc2-427b-4800-aaa3-8c4a5b58798c/crucible/regions/f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:51.225962557Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"region":"f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86","component":"worker"}
{"msg":"region f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:51.272141186Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T10:28:51.279738237Z","hostname":"oxz_crucible_fff0848b-77b2-4039-99eb-c1403142304e","pid":28210,"uri":"/crucible/0/regions/f41d41f2-ae75-40ba-a8a8-78cd8bdaeb86","method":"GET","req_id":"733e4404-e512-4053-ac3d-ae23e3dfe0c9","remote_addr":"[fd00:1122:3344:101::a]:38039","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":55,"response_code":200}