[ Mar  4 04:29:57 Enabled. ]
[ Mar  4 04:29:57 Rereading configuration. ]
[ Mar  4 04:30:00 Rereading configuration. ]
[ Mar  4 04:30:05 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:05.807831352Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994}
{"msg":"dataset: \"oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:05.808534976Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:05.808547246Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:05.808553156Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:05.822421733Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:05.933750851Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.369853218Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:35959"}
{"msg":"region 588a5c76-e606-4e95-84ca-ef6b86ccffba state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.394345141Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.397184375Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.397240056Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"uri":"/crucible/0/regions","method":"POST","req_id":"1c9ea99b-5021-4f1b-a682-6f121197afb3","remote_addr":"[fd00:1122:3344:101::c]:35959","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":13469,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.405651939Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.405670739Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"588a5c76-e606-4e95-84ca-ef6b86ccffba\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.471459788Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"region":"588a5c76-e606-4e95-84ca-ef6b86ccffba","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"588a5c76-e606-4e95-84ca-ef6b86ccffba\" \"--data\" \"/data/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.473764762Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"region":"588a5c76-e606-4e95-84ca-ef6b86ccffba","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.592362879Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"uri":"/crucible/0/regions","method":"POST","req_id":"bb186700-514b-4e85-a6a8-a1c61cc15e06","remote_addr":"[fd00:1122:3344:101::c]:35959","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":63,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.84911977Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"region":"588a5c76-e606-4e95-84ca-ef6b86ccffba","component":"worker"}
{"msg":"region 588a5c76-e606-4e95-84ca-ef6b86ccffba state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.849197222Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.849374231Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-588a5c76-e606-4e95-84ca-ef6b86ccffba","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.849707566Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-588a5c76-e606-4e95-84ca-ef6b86ccffba","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.855615046Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.872146632Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-588a5c76-e606-4e95-84ca-ef6b86ccffba","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.878057458Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.878884884Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.878945455Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.878988356Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.879022747Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.884101855Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:44.888831076Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:30:45.114977887Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"uri":"/crucible/0/regions","method":"POST","req_id":"b22363fb-fc5b-44f0-a33f-b8cda65c411d","remote_addr":"[fd00:1122:3344:101::c]:35959","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":78,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.475159051Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:35400"}
{"msg":"region 616d4646-0558-40fe-8583-7d02026ad6a7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.476017928Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.476187611Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"uri":"/crucible/0/regions","method":"POST","req_id":"8dc2e88d-22a3-4984-9f09-54ce59117e4b","remote_addr":"[fd00:1122:3344:101::c]:35400","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":921,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.476202711Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible/regions/616d4646-0558-40fe-8583-7d02026ad6a7","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.486238075Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible/regions/616d4646-0558-40fe-8583-7d02026ad6a7","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.486265675Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"616d4646-0558-40fe-8583-7d02026ad6a7\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/616d4646-0558-40fe-8583-7d02026ad6a7\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.549618577Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"region":"616d4646-0558-40fe-8583-7d02026ad6a7","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"616d4646-0558-40fe-8583-7d02026ad6a7\" \"--data\" \"/data/regions/616d4646-0558-40fe-8583-7d02026ad6a7\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.549655598Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"region":"616d4646-0558-40fe-8583-7d02026ad6a7","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.676104596Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"uri":"/crucible/0/regions","method":"POST","req_id":"a9619aeb-afc4-46f5-a9a2-d12553c535b8","remote_addr":"[fd00:1122:3344:101::c]:35400","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":66,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.748480572Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"region":"616d4646-0558-40fe-8583-7d02026ad6a7","component":"worker"}
{"msg":"region 616d4646-0558-40fe-8583-7d02026ad6a7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.748504992Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.748634935Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-616d4646-0558-40fe-8583-7d02026ad6a7","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.768658481Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-616d4646-0558-40fe-8583-7d02026ad6a7","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.779318477Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.801482584Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-616d4646-0558-40fe-8583-7d02026ad6a7","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.809259224Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/616d4646-0558-40fe-8583-7d02026ad6a7","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.809488439Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.809539769Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.80958403Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.809624721Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.817771538Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:09.826737061Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:10.398511787Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"uri":"/crucible/0/regions","method":"POST","req_id":"b76802bc-7ab8-42a6-8030-1521398eddcf","remote_addr":"[fd00:1122:3344:101::c]:35400","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":61,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.169341005Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"uri":"/crucible/0/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba","method":"GET","req_id":"2eedf7e1-1665-4fde-8508-e73de06ee981","remote_addr":"[fd00:1122:3344:101::c]:35400","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":79,"response_code":200}
{"msg":"path is \"/data/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.173041577Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"dataset is oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.180188664Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"snapshot is oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba@c83ec41c-5341-4ce4-8349-d181265001b0","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.187093728Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba\", \"c83ec41c-5341-4ce4-8349-d181265001b0\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.187117528Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.195283115Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"uri":"/crucible/0/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba/snapshots/c83ec41c-5341-4ce4-8349-d181265001b0","method":"GET","req_id":"929fdb60-2973-4c6d-ad4b-64d5be5c1685","remote_addr":"[fd00:1122:3344:101::c]:35400","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":22269,"response_code":200}
{"msg":"path is \"/data/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.199075559Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"dataset is oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.205626435Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"snapshot is oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba@c83ec41c-5341-4ce4-8349-d181265001b0","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.211688252Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba\", \"c83ec41c-5341-4ce4-8349-d181265001b0\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.211722852Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 588a5c76-e606-4e95-84ca-ef6b86ccffba-c83ec41c-5341-4ce4-8349-d181265001b0 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.219109155Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.219245518Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"uri":"/crucible/0/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba/snapshots/c83ec41c-5341-4ce4-8349-d181265001b0/run","method":"POST","req_id":"6971be14-fc1f-402b-8146-e17c6f52244d","remote_addr":"[fd00:1122:3344:101::c]:35400","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":20202,"response_code":200}
{"msg":"applying SMF actions for region 588a5c76-e606-4e95-84ca-ef6b86ccffba running snapshot c83ec41c-5341-4ce4-8349-d181265001b0 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.219278978Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-588a5c76-e606-4e95-84ca-ef6b86ccffba-c83ec41c-5341-4ce4-8349-d181265001b0","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.223401778Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-588a5c76-e606-4e95-84ca-ef6b86ccffba-c83ec41c-5341-4ce4-8349-d181265001b0","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.244093927Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-588a5c76-e606-4e95-84ca-ef6b86ccffba-c83ec41c-5341-4ce4-8349-d181265001b0","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.278722585Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba/.zfs/snapshot/c83ec41c-5341-4ce4-8349-d181265001b0","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.279084992Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.279120702Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.279164263Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.279191394Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.279242595Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.287922012Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.30024129Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"region 588a5c76-e606-4e95-84ca-ef6b86ccffba running snapshot c83ec41c-5341-4ce4-8349-d181265001b0 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.3002544Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"path is \"/data/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.565295581Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"dataset is oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.57195053Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"snapshot is oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba@c83ec41c-5341-4ce4-8349-d181265001b0","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.578397834Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba\", \"c83ec41c-5341-4ce4-8349-d181265001b0\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.578423544Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:12.584910489Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"uri":"/crucible/0/regions/588a5c76-e606-4e95-84ca-ef6b86ccffba/snapshots/c83ec41c-5341-4ce4-8349-d181265001b0/run","method":"POST","req_id":"bf7b3039-7c7f-4cb7-98b3-326700744b3e","remote_addr":"[fd00:1122:3344:101::c]:35400","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":19648,"response_code":200}
{"msg":"region f947cf0e-44bd-4bd4-a842-632cf1be5428 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.379658376Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.379832969Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"uri":"/crucible/0/regions","method":"POST","req_id":"c0f194a7-790d-4c8c-8d61-35278a04f739","remote_addr":"[fd00:1122:3344:101::c]:35400","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":233,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.37985835Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible/regions/f947cf0e-44bd-4bd4-a842-632cf1be5428","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.387012608Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible/regions/f947cf0e-44bd-4bd4-a842-632cf1be5428","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.387059829Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f947cf0e-44bd-4bd4-a842-632cf1be5428\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f947cf0e-44bd-4bd4-a842-632cf1be5428\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.448434902Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"region":"f947cf0e-44bd-4bd4-a842-632cf1be5428","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f947cf0e-44bd-4bd4-a842-632cf1be5428\" \"--data\" \"/data/regions/f947cf0e-44bd-4bd4-a842-632cf1be5428\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.448468223Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"region":"f947cf0e-44bd-4bd4-a842-632cf1be5428","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.523608152Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"uri":"/crucible/0/regions","method":"POST","req_id":"8f83d465-f2a7-4ed9-a5ef-de1e3d986067","remote_addr":"[fd00:1122:3344:101::c]:35400","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":84,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.592035762Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"region":"f947cf0e-44bd-4bd4-a842-632cf1be5428","component":"worker"}
{"msg":"region f947cf0e-44bd-4bd4-a842-632cf1be5428 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.592071932Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.592233375Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f947cf0e-44bd-4bd4-a842-632cf1be5428","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.594993429Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f947cf0e-44bd-4bd4-a842-632cf1be5428","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.606411719Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.670833141Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f947cf0e-44bd-4bd4-a842-632cf1be5428","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.679739933Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f947cf0e-44bd-4bd4-a842-632cf1be5428","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.680206852Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.680278723Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.680347555Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.680391415Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.696311833Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:18.712018925Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:19.185355033Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"uri":"/crucible/0/regions","method":"POST","req_id":"d6398279-1a40-44bf-a763-557d2396a833","remote_addr":"[fd00:1122:3344:101::c]:35400","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":70,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:48.890636337Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"uri":"/crucible/0/regions/f947cf0e-44bd-4bd4-a842-632cf1be5428","method":"GET","req_id":"7c1ba03a-8619-4243-9462-5b18e083d031","remote_addr":"[fd00:1122:3344:101::c]:35400","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":56,"response_code":200}
{"msg":"path is \"/data/regions/f947cf0e-44bd-4bd4-a842-632cf1be5428\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:48.893998731Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"dataset is oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible/regions/f947cf0e-44bd-4bd4-a842-632cf1be5428","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:48.904189978Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"region f947cf0e-44bd-4bd4-a842-632cf1be5428 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:48.911640192Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:48.911806425Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"uri":"/crucible/0/regions/f947cf0e-44bd-4bd4-a842-632cf1be5428","method":"DELETE","req_id":"3c6ee9d9-b033-46c7-b1ea-76f4712e37ad","remote_addr":"[fd00:1122:3344:101::c]:35400","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":17832,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:48.911831895Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-f947cf0e-44bd-4bd4-a842-632cf1be5428 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:48.912547159Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:48.915366714Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"uri":"/crucible/0/regions/f947cf0e-44bd-4bd4-a842-632cf1be5428","method":"GET","req_id":"51a222d4-c4ad-406e-91f0-7216fcb09776","remote_addr":"[fd00:1122:3344:101::c]:35400","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":33,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:48.922056472Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_2f7acb3a-3f1c-4289-996d-23a0d0fc0421/crucible/regions/f947cf0e-44bd-4bd4-a842-632cf1be5428\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:48.930939634Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"region":"f947cf0e-44bd-4bd4-a842-632cf1be5428","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:48.975044434Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"uri":"/crucible/0/regions/f947cf0e-44bd-4bd4-a842-632cf1be5428","method":"GET","req_id":"b45c10f8-a546-490d-80af-3ac51fcd406b","remote_addr":"[fd00:1122:3344:101::c]:35400","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":38,"response_code":200}
{"msg":"region f947cf0e-44bd-4bd4-a842-632cf1be5428 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:48.989937122Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-04T04:42:49.089008372Z","hostname":"oxz_crucible_2e7881f1-1494-48ec-8549-902113196460","pid":28994,"uri":"/crucible/0/regions/f947cf0e-44bd-4bd4-a842-632cf1be5428","method":"GET","req_id":"e699350e-b84b-45f9-8e60-8b1de0095db2","remote_addr":"[fd00:1122:3344:101::c]:35400","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":57,"response_code":200}