[ Jun 3 01:01:29 Enabled. ] [ Jun 3 01:01:29 Rereading configuration. ] [ Jun 3 01:01:31 Rereading configuration. ] [ Jun 3 01:01:34 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_6128d878-3726-46e4-aee6-3261382709f9/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: bd9a0e2abe6b6b89aec8c85f4ee57474144ed150\nCommit timestamp: 2026-05-14T15:48:00.000000000Z branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:01:34.872541792Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961} {"msg":"dataset: \"oxp_6128d878-3726-46e4-aee6-3261382709f9/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:01:34.872788977Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961} {"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:01:34.872811088Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:01:34.872820278Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:01:34.891885745Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:01:34.968030781Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.664822475Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:44303"} {"msg":"region 3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.70013817Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.704874249Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.70494976Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"uri":"/crucible/0/regions","method":"POST","req_id":"4ddfd33f-c0ba-44ed-a9fb-135ee3e62823","remote_addr":"[fd00:1122:3344:101::c]:44303","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":13840,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_6128d878-3726-46e4-aee6-3261382709f9/crucible/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.713852895Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_6128d878-3726-46e4-aee6-3261382709f9/crucible/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.713876946Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.785986727Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"region":"3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813\" \"--data\" \"/data/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.788140282Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"region":"3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.925675087Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"region":"3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","component":"worker"} {"msg":"region 3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.925750728Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.925971513Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.928156238Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.932795695Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.944415417Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"uri":"/crucible/0/regions","method":"POST","req_id":"d8cbb281-9e38-459d-b470-3f4946f8d0c6","remote_addr":"[fd00:1122:3344:101::c]:44303","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":72,"response_code":200} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.945200503Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.947114493Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.947796867Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.947857598Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.947895659Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.94794368Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.957625322Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.960628214Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.323949585Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:32954"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.324068107Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"uri":"/crucible/0/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","method":"GET","req_id":"d3d7df88-2b32-46cd-b618-b73f3248295a","remote_addr":"[fd00:1122:3344:101::c]:32954","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":59,"response_code":200} {"msg":"path is \"/data/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.327507089Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"datafile"} {"msg":"dataset is oxp_6128d878-3726-46e4-aee6-3261382709f9/crucible/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.333277499Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"datafile"} {"msg":"snapshot is oxp_6128d878-3726-46e4-aee6-3261382709f9/crucible/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813@c89e0aa1-5551-4813-9507-8c80561a6b6f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.338030288Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_6128d878-3726-46e4-aee6-3261382709f9/crucible/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813\", \"c89e0aa1-5551-4813-9507-8c80561a6b6f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.338063139Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.347516716Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"uri":"/crucible/0/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813/snapshots/c89e0aa1-5551-4813-9507-8c80561a6b6f","method":"GET","req_id":"9ca2106b-2c97-4701-bbce-7970b06fbe85","remote_addr":"[fd00:1122:3344:101::c]:32954","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":20041,"response_code":200} {"msg":"path is \"/data/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.351415147Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"datafile"} {"msg":"dataset is oxp_6128d878-3726-46e4-aee6-3261382709f9/crucible/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.357123736Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"datafile"} {"msg":"snapshot is oxp_6128d878-3726-46e4-aee6-3261382709f9/crucible/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813@c89e0aa1-5551-4813-9507-8c80561a6b6f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.363212882Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_6128d878-3726-46e4-aee6-3261382709f9/crucible/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813\", \"c89e0aa1-5551-4813-9507-8c80561a6b6f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.363239293Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813-c89e0aa1-5551-4813-9507-8c80561a6b6f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.369022034Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"datafile"} {"msg":"applying SMF actions for region 3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813 running snapshot c89e0aa1-5551-4813-9507-8c80561a6b6f (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.371846492Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.371864183Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"uri":"/crucible/0/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813/snapshots/c89e0aa1-5551-4813-9507-8c80561a6b6f/run","method":"POST","req_id":"e8eee004-a753-4c66-a3a4-215d18532f49","remote_addr":"[fd00:1122:3344:101::c]:32954","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":20478,"response_code":200} {"msg":"existing directory value /dev/null does not match /data/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.380653736Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.380730177Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.380804319Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.380811789Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.38084927Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.380893611Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.380989853Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.381038464Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.381082765Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.381123635Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.386529818Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813-c89e0aa1-5551-4813-9507-8c80561a6b6f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.389740935Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813-c89e0aa1-5551-4813-9507-8c80561a6b6f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.395573266Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813-c89e0aa1-5551-4813-9507-8c80561a6b6f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.402547792Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813/.zfs/snapshot/c89e0aa1-5551-4813-9507-8c80561a6b6f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.402807167Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.402859438Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.402895729Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.40293413Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.402983371Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.404677246Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.407584797Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"region 3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813 running snapshot c89e0aa1-5551-4813-9507-8c80561a6b6f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.407611967Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"datafile"} {"msg":"path is \"/data/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.540521385Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"datafile"} {"msg":"dataset is oxp_6128d878-3726-46e4-aee6-3261382709f9/crucible/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.546816966Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"datafile"} {"msg":"snapshot is oxp_6128d878-3726-46e4-aee6-3261382709f9/crucible/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813@c89e0aa1-5551-4813-9507-8c80561a6b6f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.55276082Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_6128d878-3726-46e4-aee6-3261382709f9/crucible/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813\", \"c89e0aa1-5551-4813-9507-8c80561a6b6f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.55279273Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:39.557274554Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"uri":"/crucible/0/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813/snapshots/c89e0aa1-5551-4813-9507-8c80561a6b6f/run","method":"POST","req_id":"d6e7f200-1085-4f3d-ad2a-ae3784700f28","remote_addr":"[fd00:1122:3344:101::c]:32954","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":16796,"response_code":200} {"msg":"region 033b57fd-b84f-429c-a10b-e5b99453e443 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.641448335Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.641650399Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.64168617Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"uri":"/crucible/0/regions","method":"POST","req_id":"debb872e-ff3f-4976-8363-c444a79f3b2a","remote_addr":"[fd00:1122:3344:101::c]:32954","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":260,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_6128d878-3726-46e4-aee6-3261382709f9/crucible/regions/033b57fd-b84f-429c-a10b-e5b99453e443","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.648280357Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_6128d878-3726-46e4-aee6-3261382709f9/crucible/regions/033b57fd-b84f-429c-a10b-e5b99453e443","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.648296468Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"033b57fd-b84f-429c-a10b-e5b99453e443\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/033b57fd-b84f-429c-a10b-e5b99453e443\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.697672016Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"region":"033b57fd-b84f-429c-a10b-e5b99453e443","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"033b57fd-b84f-429c-a10b-e5b99453e443\" \"--data\" \"/data/regions/033b57fd-b84f-429c-a10b-e5b99453e443\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.697732187Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"region":"033b57fd-b84f-429c-a10b-e5b99453e443","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.744817017Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"region":"033b57fd-b84f-429c-a10b-e5b99453e443","component":"worker"} {"msg":"region 033b57fd-b84f-429c-a10b-e5b99453e443 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.744845707Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.744982601Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-033b57fd-b84f-429c-a10b-e5b99453e443","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.745336568Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-033b57fd-b84f-429c-a10b-e5b99453e443","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.747212387Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.750565587Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-033b57fd-b84f-429c-a10b-e5b99453e443","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.752069878Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/033b57fd-b84f-429c-a10b-e5b99453e443","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.752417705Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.752455966Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.752504717Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.752540908Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.754223393Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.756991611Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.757093603Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.757171824Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.757179215Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.757215435Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.757256976Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.759596005Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.759650036Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.759697577Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.759740938Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.761595706Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.763487646Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.97688924Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"uri":"/crucible/0/regions","method":"POST","req_id":"f98fdeab-8460-4e9a-bc5e-a0e282e14150","remote_addr":"[fd00:1122:3344:101::c]:32954","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":68,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.397396995Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"uri":"/crucible/0/regions/033b57fd-b84f-429c-a10b-e5b99453e443","method":"GET","req_id":"6212c0b3-199e-4ef6-952b-473f2dd6a44f","remote_addr":"[fd00:1122:3344:101::c]:32954","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":70,"response_code":200} {"msg":"path is \"/data/regions/033b57fd-b84f-429c-a10b-e5b99453e443\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.400742895Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"datafile"} {"msg":"dataset is oxp_6128d878-3726-46e4-aee6-3261382709f9/crucible/regions/033b57fd-b84f-429c-a10b-e5b99453e443","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.411101051Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"datafile"} {"msg":"region 033b57fd-b84f-429c-a10b-e5b99453e443 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.418080246Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.418248499Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.41827938Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"uri":"/crucible/0/regions/033b57fd-b84f-429c-a10b-e5b99453e443","method":"DELETE","req_id":"254e629f-cafe-4222-9a17-872189fec77a","remote_addr":"[fd00:1122:3344:101::c]:32954","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":17556,"response_code":204} {"msg":"disabling downstairs instance: downstairs-033b57fd-b84f-429c-a10b-e5b99453e443 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.418799811Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.421728442Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"uri":"/crucible/0/regions/033b57fd-b84f-429c-a10b-e5b99453e443","method":"GET","req_id":"be921545-7c64-4d21-bed4-c264081233ed","remote_addr":"[fd00:1122:3344:101::c]:32954","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":34,"response_code":200} {"msg":"existing directory value /dev/null does not match /data/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.422066399Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.422144911Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.422223352Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.422236473Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.422268903Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.422312284Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3c56fd9a-e5d5-4608-b1e5-2c5f6e7dd813","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.422627661Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.422682862Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.422727123Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.422769534Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.425607813Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.427079774Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_6128d878-3726-46e4-aee6-3261382709f9/crucible/regions/033b57fd-b84f-429c-a10b-e5b99453e443\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.434935157Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"region":"033b57fd-b84f-429c-a10b-e5b99453e443","component":"worker"} {"msg":"region 033b57fd-b84f-429c-a10b-e5b99453e443 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.477449412Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.536509642Z","hostname":"oxz_crucible_616c0a6f-a17f-4d0b-bfb2-cbfc65db11c6","pid":26961,"uri":"/crucible/0/regions/033b57fd-b84f-429c-a10b-e5b99453e443","method":"GET","req_id":"9f932a65-0e07-4e00-82ce-95827895fe00","remote_addr":"[fd00:1122:3344:101::c]:32954","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":49,"response_code":200}