[ Apr  7 02:47:30 Enabled. ]
[ Apr  7 02:47:30 Rereading configuration. ]
[ Apr  7 02:47:32 Rereading configuration. ]
[ Apr  7 02:47:37 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_5f384caf-75dd-418d-9c96-1b05f9c84a7c/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:47:37.388402631Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983}
{"msg":"dataset: \"oxp_5f384caf-75dd-418d-9c96-1b05f9c84a7c/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:47:37.389154307Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:47:37.389164137Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:47:37.389169247Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:47:37.405351119Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:47:37.649900969Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:21.636725829Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:48974"}
{"msg":"region f5592633-5f77-439b-96b7-73517f74fd96 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:21.66333036Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:21.666114769Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:21.66616504Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"uri":"/crucible/0/regions","method":"POST","req_id":"71dd8c9d-8d40-4e35-a17b-885044f23af5","remote_addr":"[fd00:1122:3344:101::b]:48974","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":14279,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_5f384caf-75dd-418d-9c96-1b05f9c84a7c/crucible/regions/f5592633-5f77-439b-96b7-73517f74fd96","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:21.673901643Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_5f384caf-75dd-418d-9c96-1b05f9c84a7c/crucible/regions/f5592633-5f77-439b-96b7-73517f74fd96","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:21.673917973Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f5592633-5f77-439b-96b7-73517f74fd96\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f5592633-5f77-439b-96b7-73517f74fd96\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:21.736873532Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"region":"f5592633-5f77-439b-96b7-73517f74fd96","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f5592633-5f77-439b-96b7-73517f74fd96\" \"--data\" \"/data/regions/f5592633-5f77-439b-96b7-73517f74fd96\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:21.74011179Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"region":"f5592633-5f77-439b-96b7-73517f74fd96","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:21.83440481Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"uri":"/crucible/0/regions","method":"POST","req_id":"cb6eff76-8a71-4073-8934-111f21bb7474","remote_addr":"[fd00:1122:3344:101::b]:48974","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":61,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:22.106574743Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"region":"f5592633-5f77-439b-96b7-73517f74fd96","component":"worker"}
{"msg":"region f5592633-5f77-439b-96b7-73517f74fd96 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:22.106659845Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:22.106854199Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f5592633-5f77-439b-96b7-73517f74fd96","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:22.107198646Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f5592633-5f77-439b-96b7-73517f74fd96","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:22.113821666Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:22.128025626Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f5592633-5f77-439b-96b7-73517f74fd96","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:22.13153402Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f5592633-5f77-439b-96b7-73517f74fd96","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:22.132276915Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:22.132336417Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:22.132385478Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:22.132419878Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:22.136823221Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:22.142210925Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:48:22.48652782Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"uri":"/crucible/0/regions","method":"POST","req_id":"888d876d-e597-4868-a509-8475fcd8fc3f","remote_addr":"[fd00:1122:3344:101::b]:48974","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":59,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.217597668Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:55358"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.218191561Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"uri":"/crucible/0/regions/f5592633-5f77-439b-96b7-73517f74fd96","method":"GET","req_id":"339bdfe2-84b9-41a8-874f-bb81861f0990","remote_addr":"[fd00:1122:3344:101::b]:55358","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":72,"response_code":200}
{"msg":"path is \"/data/regions/f5592633-5f77-439b-96b7-73517f74fd96\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.220908958Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"datafile"}
{"msg":"dataset is oxp_5f384caf-75dd-418d-9c96-1b05f9c84a7c/crucible/regions/f5592633-5f77-439b-96b7-73517f74fd96","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.227975427Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"datafile"}
{"msg":"snapshot is oxp_5f384caf-75dd-418d-9c96-1b05f9c84a7c/crucible/regions/f5592633-5f77-439b-96b7-73517f74fd96@d85562e7-4d7e-4a5a-9bf5-4c1f300d0489","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.233848782Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_5f384caf-75dd-418d-9c96-1b05f9c84a7c/crucible/regions/f5592633-5f77-439b-96b7-73517f74fd96\", \"d85562e7-4d7e-4a5a-9bf5-4c1f300d0489\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.233879112Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.244814033Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"uri":"/crucible/0/regions/f5592633-5f77-439b-96b7-73517f74fd96/snapshots/d85562e7-4d7e-4a5a-9bf5-4c1f300d0489","method":"GET","req_id":"6e8ed26c-dc34-463b-b16f-e035f2aba333","remote_addr":"[fd00:1122:3344:101::b]:55358","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":23920,"response_code":200}
{"msg":"path is \"/data/regions/f5592633-5f77-439b-96b7-73517f74fd96\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.248105392Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"datafile"}
{"msg":"dataset is oxp_5f384caf-75dd-418d-9c96-1b05f9c84a7c/crucible/regions/f5592633-5f77-439b-96b7-73517f74fd96","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.25464728Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"datafile"}
{"msg":"snapshot is oxp_5f384caf-75dd-418d-9c96-1b05f9c84a7c/crucible/regions/f5592633-5f77-439b-96b7-73517f74fd96@d85562e7-4d7e-4a5a-9bf5-4c1f300d0489","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.260735509Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_5f384caf-75dd-418d-9c96-1b05f9c84a7c/crucible/regions/f5592633-5f77-439b-96b7-73517f74fd96\", \"d85562e7-4d7e-4a5a-9bf5-4c1f300d0489\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.260757379Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot f5592633-5f77-439b-96b7-73517f74fd96-d85562e7-4d7e-4a5a-9bf5-4c1f300d0489 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.268405501Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.268997363Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"uri":"/crucible/0/regions/f5592633-5f77-439b-96b7-73517f74fd96/snapshots/d85562e7-4d7e-4a5a-9bf5-4c1f300d0489/run","method":"POST","req_id":"61bf42f4-503f-45eb-98a5-7bd5fd9c40a3","remote_addr":"[fd00:1122:3344:101::b]:55358","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":20893,"response_code":200}
{"msg":"applying SMF actions for region f5592633-5f77-439b-96b7-73517f74fd96 running snapshot d85562e7-4d7e-4a5a-9bf5-4c1f300d0489 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.269013973Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-f5592633-5f77-439b-96b7-73517f74fd96-d85562e7-4d7e-4a5a-9bf5-4c1f300d0489","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.289015455Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-f5592633-5f77-439b-96b7-73517f74fd96-d85562e7-4d7e-4a5a-9bf5-4c1f300d0489","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.297789931Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-f5592633-5f77-439b-96b7-73517f74fd96-d85562e7-4d7e-4a5a-9bf5-4c1f300d0489","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.321229445Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f5592633-5f77-439b-96b7-73517f74fd96/.zfs/snapshot/d85562e7-4d7e-4a5a-9bf5-4c1f300d0489","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.321594573Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.321636834Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.321673665Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.321709185Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.321765366Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.328039569Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.33519104Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"region f5592633-5f77-439b-96b7-73517f74fd96 running snapshot d85562e7-4d7e-4a5a-9bf5-4c1f300d0489 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.33520766Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"datafile"}
{"msg":"path is \"/data/regions/f5592633-5f77-439b-96b7-73517f74fd96\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.459612535Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"datafile"}
{"msg":"dataset is oxp_5f384caf-75dd-418d-9c96-1b05f9c84a7c/crucible/regions/f5592633-5f77-439b-96b7-73517f74fd96","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.465755995Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"datafile"}
{"msg":"snapshot is oxp_5f384caf-75dd-418d-9c96-1b05f9c84a7c/crucible/regions/f5592633-5f77-439b-96b7-73517f74fd96@d85562e7-4d7e-4a5a-9bf5-4c1f300d0489","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.472234132Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_5f384caf-75dd-418d-9c96-1b05f9c84a7c/crucible/regions/f5592633-5f77-439b-96b7-73517f74fd96\", \"d85562e7-4d7e-4a5a-9bf5-4c1f300d0489\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.472256962Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:58:58.478984254Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"uri":"/crucible/0/regions/f5592633-5f77-439b-96b7-73517f74fd96/snapshots/d85562e7-4d7e-4a5a-9bf5-4c1f300d0489/run","method":"POST","req_id":"f9060da8-b393-4e96-a635-238e0350ea8f","remote_addr":"[fd00:1122:3344:101::b]:55358","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":19432,"response_code":200}
{"msg":"region b75bb159-eeaa-478c-9e7c-4b149cb4769f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.330656309Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.330824742Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"uri":"/crucible/0/regions","method":"POST","req_id":"9cca685b-dc64-4689-a879-61ab1908276d","remote_addr":"[fd00:1122:3344:101::b]:55358","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":216,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.330851973Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_5f384caf-75dd-418d-9c96-1b05f9c84a7c/crucible/regions/b75bb159-eeaa-478c-9e7c-4b149cb4769f","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.338100426Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_5f384caf-75dd-418d-9c96-1b05f9c84a7c/crucible/regions/b75bb159-eeaa-478c-9e7c-4b149cb4769f","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.338118826Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b75bb159-eeaa-478c-9e7c-4b149cb4769f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b75bb159-eeaa-478c-9e7c-4b149cb4769f\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.389515081Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"region":"b75bb159-eeaa-478c-9e7c-4b149cb4769f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b75bb159-eeaa-478c-9e7c-4b149cb4769f\" \"--data\" \"/data/regions/b75bb159-eeaa-478c-9e7c-4b149cb4769f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.389550321Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"region":"b75bb159-eeaa-478c-9e7c-4b149cb4769f","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.460887577Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"region":"b75bb159-eeaa-478c-9e7c-4b149cb4769f","component":"worker"}
{"msg":"region b75bb159-eeaa-478c-9e7c-4b149cb4769f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.460918518Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.46102344Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b75bb159-eeaa-478c-9e7c-4b149cb4769f","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.461386317Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b75bb159-eeaa-478c-9e7c-4b149cb4769f","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.473456052Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.479915008Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b75bb159-eeaa-478c-9e7c-4b149cb4769f","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.483082385Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b75bb159-eeaa-478c-9e7c-4b149cb4769f","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.483280619Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.48332017Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.483354721Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.483389262Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.486678511Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.497252194Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:05.593828757Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"uri":"/crucible/0/regions","method":"POST","req_id":"7045d1f1-5a54-4a9d-a60d-17f578a7b0e1","remote_addr":"[fd00:1122:3344:101::b]:55358","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":68,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:31.763416889Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"uri":"/crucible/0/regions/b75bb159-eeaa-478c-9e7c-4b149cb4769f","method":"GET","req_id":"ba29ee58-af9f-4c28-ad28-bd5b23770ef1","remote_addr":"[fd00:1122:3344:101::b]:55358","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":60,"response_code":200}
{"msg":"path is \"/data/regions/b75bb159-eeaa-478c-9e7c-4b149cb4769f\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:31.766339621Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"datafile"}
{"msg":"dataset is oxp_5f384caf-75dd-418d-9c96-1b05f9c84a7c/crucible/regions/b75bb159-eeaa-478c-9e7c-4b149cb4769f","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:31.775657688Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"datafile"}
{"msg":"region b75bb159-eeaa-478c-9e7c-4b149cb4769f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:31.783403461Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:31.783583405Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"uri":"/crucible/0/regions/b75bb159-eeaa-478c-9e7c-4b149cb4769f","method":"DELETE","req_id":"8228670e-084d-45bb-990a-cee2c10e64d0","remote_addr":"[fd00:1122:3344:101::b]:55358","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":17260,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:31.783640166Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-b75bb159-eeaa-478c-9e7c-4b149cb4769f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:31.784234709Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:31.786593849Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"uri":"/crucible/0/regions/b75bb159-eeaa-478c-9e7c-4b149cb4769f","method":"GET","req_id":"6331eaa4-e256-4a57-9cb1-a7c325ff6431","remote_addr":"[fd00:1122:3344:101::b]:55358","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":43,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:31.800620714Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_5f384caf-75dd-418d-9c96-1b05f9c84a7c/crucible/regions/b75bb159-eeaa-478c-9e7c-4b149cb4769f\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:31.808888809Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"region":"b75bb159-eeaa-478c-9e7c-4b149cb4769f","component":"worker"}
{"msg":"region b75bb159-eeaa-478c-9e7c-4b149cb4769f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:31.870478999Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T02:59:31.885996776Z","hostname":"oxz_crucible_75529dad-f24c-4ff0-804e-0df006c9524b","pid":23983,"uri":"/crucible/0/regions/b75bb159-eeaa-478c-9e7c-4b149cb4769f","method":"GET","req_id":"fdc0ae6d-3e86-419f-9523-45e1d927699a","remote_addr":"[fd00:1122:3344:101::b]:55358","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":55,"response_code":200}