[ May  8 03:00:06 Enabled. ]
[ May  8 03:00:06 Rereading configuration. ]
[ May  8 03:00:07 Rereading configuration. ]
[ May  8 03:00:10 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_8cd0f983-f129-4a01-ab5a-495470988dc8/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:00:10.380647588Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728}
{"msg":"dataset: \"oxp_8cd0f983-f129-4a01-ab5a-495470988dc8/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:00:10.381472033Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:00:10.381486593Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:00:10.381497304Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:00:10.402559933Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:00:10.470630031Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.749337792Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:59162"}
{"msg":"region 537b346e-285f-49e3-925d-3c21b1e0ba47 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.793461483Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.794874326Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.794895886Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"uri":"/crucible/0/regions","method":"POST","req_id":"145c2232-de37-4089-997e-fb93be86b5e1","remote_addr":"[fd00:1122:3344:101::b]:59162","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":18949,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_8cd0f983-f129-4a01-ab5a-495470988dc8/crucible/regions/537b346e-285f-49e3-925d-3c21b1e0ba47","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.803098796Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8cd0f983-f129-4a01-ab5a-495470988dc8/crucible/regions/537b346e-285f-49e3-925d-3c21b1e0ba47","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.803113046Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"537b346e-285f-49e3-925d-3c21b1e0ba47\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/537b346e-285f-49e3-925d-3c21b1e0ba47\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.842551294Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"region":"537b346e-285f-49e3-925d-3c21b1e0ba47","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"537b346e-285f-49e3-925d-3c21b1e0ba47\" \"--data\" \"/data/regions/537b346e-285f-49e3-925d-3c21b1e0ba47\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:17.844802832Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"region":"537b346e-285f-49e3-925d-3c21b1e0ba47","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:18.129911894Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"uri":"/crucible/0/regions","method":"POST","req_id":"a0bb87bd-a3a8-4bf2-a61e-c651292c0e38","remote_addr":"[fd00:1122:3344:101::b]:59162","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:18.156579504Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"region":"537b346e-285f-49e3-925d-3c21b1e0ba47","component":"worker"}
{"msg":"region 537b346e-285f-49e3-925d-3c21b1e0ba47 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:18.156626966Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:18.156726829Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-537b346e-285f-49e3-925d-3c21b1e0ba47","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:18.15711067Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-537b346e-285f-49e3-925d-3c21b1e0ba47","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:18.162335169Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:18.170828237Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-537b346e-285f-49e3-925d-3c21b1e0ba47","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:18.171985592Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/537b346e-285f-49e3-925d-3c21b1e0ba47","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:18.172726155Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:18.172787827Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:18.172827678Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:18.172863639Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:18.17486711Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:18.177083447Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:18.659274447Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"uri":"/crucible/0/regions","method":"POST","req_id":"05a6c7e9-9845-4552-b44b-615ac41f6fdc","remote_addr":"[fd00:1122:3344:101::b]:59162","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":70,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.526063009Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:64611"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.526154761Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"uri":"/crucible/0/regions/537b346e-285f-49e3-925d-3c21b1e0ba47","method":"GET","req_id":"d0a8f6f2-a72b-4621-b76a-28d7a43268a6","remote_addr":"[fd00:1122:3344:101::b]:64611","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":53,"response_code":200}
{"msg":"path is \"/data/regions/537b346e-285f-49e3-925d-3c21b1e0ba47\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.529583146Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"datafile"}
{"msg":"dataset is oxp_8cd0f983-f129-4a01-ab5a-495470988dc8/crucible/regions/537b346e-285f-49e3-925d-3c21b1e0ba47","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.535753303Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"datafile"}
{"msg":"snapshot is oxp_8cd0f983-f129-4a01-ab5a-495470988dc8/crucible/regions/537b346e-285f-49e3-925d-3c21b1e0ba47@defa2aee-9ac4-419c-86ae-2ad870019525","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.54092327Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8cd0f983-f129-4a01-ab5a-495470988dc8/crucible/regions/537b346e-285f-49e3-925d-3c21b1e0ba47\", \"defa2aee-9ac4-419c-86ae-2ad870019525\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.540948271Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.551921454Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"uri":"/crucible/0/regions/537b346e-285f-49e3-925d-3c21b1e0ba47/snapshots/defa2aee-9ac4-419c-86ae-2ad870019525","method":"GET","req_id":"ad132a42-e1b7-46f4-a034-709274aa1d7a","remote_addr":"[fd00:1122:3344:101::b]:64611","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":22359,"response_code":200}
{"msg":"path is \"/data/regions/537b346e-285f-49e3-925d-3c21b1e0ba47\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.556091461Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"datafile"}
{"msg":"dataset is oxp_8cd0f983-f129-4a01-ab5a-495470988dc8/crucible/regions/537b346e-285f-49e3-925d-3c21b1e0ba47","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.561539956Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"datafile"}
{"msg":"snapshot is oxp_8cd0f983-f129-4a01-ab5a-495470988dc8/crucible/regions/537b346e-285f-49e3-925d-3c21b1e0ba47@defa2aee-9ac4-419c-86ae-2ad870019525","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.566475406Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8cd0f983-f129-4a01-ab5a-495470988dc8/crucible/regions/537b346e-285f-49e3-925d-3c21b1e0ba47\", \"defa2aee-9ac4-419c-86ae-2ad870019525\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.566492767Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 537b346e-285f-49e3-925d-3c21b1e0ba47-defa2aee-9ac4-419c-86ae-2ad870019525 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.573091378Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"datafile"}
{"msg":"applying SMF actions for region 537b346e-285f-49e3-925d-3c21b1e0ba47 running snapshot defa2aee-9ac4-419c-86ae-2ad870019525 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.57383017Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.573856261Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"uri":"/crucible/0/regions/537b346e-285f-49e3-925d-3c21b1e0ba47/snapshots/defa2aee-9ac4-419c-86ae-2ad870019525/run","method":"POST","req_id":"73ee47b9-3b26-47db-8db4-8e4726861ae4","remote_addr":"[fd00:1122:3344:101::b]:64611","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":17762,"response_code":200}
{"msg":"creating missing snapshot instance snapshot-537b346e-285f-49e3-925d-3c21b1e0ba47-defa2aee-9ac4-419c-86ae-2ad870019525","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.588101143Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-537b346e-285f-49e3-925d-3c21b1e0ba47-defa2aee-9ac4-419c-86ae-2ad870019525","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.591356682Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-537b346e-285f-49e3-925d-3c21b1e0ba47-defa2aee-9ac4-419c-86ae-2ad870019525","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.598963744Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/537b346e-285f-49e3-925d-3c21b1e0ba47/.zfs/snapshot/defa2aee-9ac4-419c-86ae-2ad870019525","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.599233092Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.599279183Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.599315134Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.599352955Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.599403887Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.601669506Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.604728528Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"region 537b346e-285f-49e3-925d-3c21b1e0ba47 running snapshot defa2aee-9ac4-419c-86ae-2ad870019525 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.604738879Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"datafile"}
{"msg":"path is \"/data/regions/537b346e-285f-49e3-925d-3c21b1e0ba47\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.706984145Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"datafile"}
{"msg":"dataset is oxp_8cd0f983-f129-4a01-ab5a-495470988dc8/crucible/regions/537b346e-285f-49e3-925d-3c21b1e0ba47","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.712983928Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"datafile"}
{"msg":"snapshot is oxp_8cd0f983-f129-4a01-ab5a-495470988dc8/crucible/regions/537b346e-285f-49e3-925d-3c21b1e0ba47@defa2aee-9ac4-419c-86ae-2ad870019525","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.718963079Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8cd0f983-f129-4a01-ab5a-495470988dc8/crucible/regions/537b346e-285f-49e3-925d-3c21b1e0ba47\", \"defa2aee-9ac4-419c-86ae-2ad870019525\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.71899889Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:17.724621481Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"uri":"/crucible/0/regions/537b346e-285f-49e3-925d-3c21b1e0ba47/snapshots/defa2aee-9ac4-419c-86ae-2ad870019525/run","method":"POST","req_id":"c0cf9272-81b2-41fd-89d9-dc52480c2c31","remote_addr":"[fd00:1122:3344:101::b]:64611","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":17691,"response_code":200}
{"msg":"region 3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.367989488Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.368176344Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"uri":"/crucible/0/regions","method":"POST","req_id":"b9f6ae58-3e86-4ef5-846a-51e808a2e222","remote_addr":"[fd00:1122:3344:101::b]:64611","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":218,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.368220555Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_8cd0f983-f129-4a01-ab5a-495470988dc8/crucible/regions/3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.375225918Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8cd0f983-f129-4a01-ab5a-495470988dc8/crucible/regions/3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.375265859Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.415337476Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"region":"3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda\" \"--data\" \"/data/regions/3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.415403029Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"region":"3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.560205918Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"region":"3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda","component":"worker"}
{"msg":"region 3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.560262439Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.560435565Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.560985731Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.570746878Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:56415"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.570969035Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"uri":"/crucible/0/regions/3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda","method":"GET","req_id":"89eac038-4388-4e8f-b480-854d79d3e47b","remote_addr":"[fd00:1122:3344:101::b]:56415","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":63,"response_code":200}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.580558006Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.586183657Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.588281621Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.588617471Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.588669402Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.588755005Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.588796966Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.591167098Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.60044161Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.660296698Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45431"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.660534666Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"uri":"/crucible/0/regions/3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda","method":"GET","req_id":"90f754e5-a4c5-43e6-abc4-c9d7a6ffd04f","remote_addr":"[fd00:1122:3344:101::a]:45431","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":65,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.698366295Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:57368"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.701261233Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"uri":"/crucible/0/regions/3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda","method":"GET","req_id":"cd5ab854-d802-4da2-8c0c-d11973bdef44","remote_addr":"[fd00:1122:3344:101::c]:57368","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":67,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.721715235Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"uri":"/crucible/0/regions","method":"POST","req_id":"97e20787-cb27-4cd0-a04f-1adafbacd986","remote_addr":"[fd00:1122:3344:101::b]:64611","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":96,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.708230333Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"uri":"/crucible/0/regions/3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda","method":"GET","req_id":"5fe1af50-1722-45b6-a378-5ca4916b9fef","remote_addr":"[fd00:1122:3344:101::b]:64611","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":52,"response_code":200}
{"msg":"path is \"/data/regions/3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.712109501Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"datafile"}
{"msg":"dataset is oxp_8cd0f983-f129-4a01-ab5a-495470988dc8/crucible/regions/3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.721239659Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"datafile"}
{"msg":"region 3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.729383716Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.729580302Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.729590872Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"uri":"/crucible/0/regions/3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda","method":"DELETE","req_id":"558bdb33-7aa6-44b0-b646-ec5b8f589d8f","remote_addr":"[fd00:1122:3344:101::b]:64611","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":17498,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.730849641Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.734580394Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"uri":"/crucible/0/regions/3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda","method":"GET","req_id":"740a1a53-20a0-4428-b373-01996d1b91d4","remote_addr":"[fd00:1122:3344:101::b]:64611","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":70,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.735559294Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_8cd0f983-f129-4a01-ab5a-495470988dc8/crucible/regions/3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.744798884Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"region":"3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda","component":"worker"}
{"msg":"region 3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.790633437Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.823562287Z","hostname":"oxz_crucible_508f9cf4-85fc-4d44-b499-7897e62211b1","pid":25728,"uri":"/crucible/0/regions/3fa6fff7-6117-4d3e-a09a-ebf07bf6ceda","method":"GET","req_id":"64e14478-61db-4ec3-a3b3-8baa7ef2cf75","remote_addr":"[fd00:1122:3344:101::b]:64611","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":54,"response_code":200}