[ Mar 11 22:15:48 Enabled. ]
[ Mar 11 22:15:48 Rereading configuration. ]
[ Mar 11 22:15:50 Rereading configuration. ]
[ Mar 11 22:15:55 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:15:55.708466074Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656}
{"msg":"dataset: \"oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:15:55.709223328Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:15:55.709233298Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:15:55.709238398Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:15:55.722623162Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:15:55.862218204Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:25.578699713Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:50600"}
{"msg":"region 94ab338b-919c-4baa-8f60-0b08f33fde3b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:25.610567576Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:25.613043693Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:25.613103195Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"uri":"/crucible/0/regions","method":"POST","req_id":"6f4475bc-6e56-4a32-a866-5b24b3ef1a35","remote_addr":"[fd00:1122:3344:101::b]:50600","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":18546,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:25.622456141Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:25.622468942Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"94ab338b-919c-4baa-8f60-0b08f33fde3b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:25.713629587Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"region":"94ab338b-919c-4baa-8f60-0b08f33fde3b","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"94ab338b-919c-4baa-8f60-0b08f33fde3b\" \"--data\" \"/data/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:25.715833479Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"region":"94ab338b-919c-4baa-8f60-0b08f33fde3b","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:25.931476851Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"uri":"/crucible/0/regions","method":"POST","req_id":"6f60d648-3484-4c8d-8be4-ded8169e3b56","remote_addr":"[fd00:1122:3344:101::b]:50600","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":67,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:26.103514907Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"region":"94ab338b-919c-4baa-8f60-0b08f33fde3b","component":"worker"}
{"msg":"region 94ab338b-919c-4baa-8f60-0b08f33fde3b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:26.103586659Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:26.103786503Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-94ab338b-919c-4baa-8f60-0b08f33fde3b","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:26.104127179Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-94ab338b-919c-4baa-8f60-0b08f33fde3b","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:26.110034571Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:26.126665126Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-94ab338b-919c-4baa-8f60-0b08f33fde3b","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:26.129793465Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:26.130533199Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:26.13060073Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:26.130633101Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:26.130688232Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:26.134886421Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:26.139737953Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:16:26.349678827Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"uri":"/crucible/0/regions","method":"POST","req_id":"d9754425-73e0-49a6-84ed-f0bc1c030b42","remote_addr":"[fd00:1122:3344:101::b]:50600","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":64,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.583242699Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:52324"}
{"msg":"region 47e47cf5-d22c-45c2-b72d-38d7db4def05 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.583518984Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.584074515Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.584085995Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"uri":"/crucible/0/regions","method":"POST","req_id":"d7c5b1f6-e0f8-4da4-8e3a-4239f9f824cd","remote_addr":"[fd00:1122:3344:101::b]:52324","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":757,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible/regions/47e47cf5-d22c-45c2-b72d-38d7db4def05","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.593026044Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible/regions/47e47cf5-d22c-45c2-b72d-38d7db4def05","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.593053745Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"47e47cf5-d22c-45c2-b72d-38d7db4def05\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/47e47cf5-d22c-45c2-b72d-38d7db4def05\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.6657264Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"region":"47e47cf5-d22c-45c2-b72d-38d7db4def05","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"47e47cf5-d22c-45c2-b72d-38d7db4def05\" \"--data\" \"/data/regions/47e47cf5-d22c-45c2-b72d-38d7db4def05\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.665761891Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"region":"47e47cf5-d22c-45c2-b72d-38d7db4def05","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.816366392Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"region":"47e47cf5-d22c-45c2-b72d-38d7db4def05","component":"worker"}
{"msg":"region 47e47cf5-d22c-45c2-b72d-38d7db4def05 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.816398692Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.816494004Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-47e47cf5-d22c-45c2-b72d-38d7db4def05","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.81682155Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-47e47cf5-d22c-45c2-b72d-38d7db4def05","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.834959494Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.865717286Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-47e47cf5-d22c-45c2-b72d-38d7db4def05","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.873819949Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/47e47cf5-d22c-45c2-b72d-38d7db4def05","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.874136325Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.874184856Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.874225837Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.874260788Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.881523585Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.901588655Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"uri":"/crucible/0/regions","method":"POST","req_id":"5a017e49-c226-4475-ad82-c7c99c00eae5","remote_addr":"[fd00:1122:3344:101::b]:52324","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":67,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:05.906164961Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.074509887Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"uri":"/crucible/0/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b","method":"GET","req_id":"b352a35c-726f-4a41-99d8-89d946e64928","remote_addr":"[fd00:1122:3344:101::b]:52324","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":80,"response_code":200}
{"msg":"path is \"/data/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.078220347Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"dataset is oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.084596498Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"snapshot is oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b@a24c7a39-7237-474e-ad7f-0f7e0e35068f","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.089796917Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b\", \"a24c7a39-7237-474e-ad7f-0f7e0e35068f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.089818077Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.098829288Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"uri":"/crucible/0/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b/snapshots/a24c7a39-7237-474e-ad7f-0f7e0e35068f","method":"GET","req_id":"0f8aed26-ee9a-49d8-8488-d3897d14121f","remote_addr":"[fd00:1122:3344:101::b]:52324","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":20649,"response_code":200}
{"msg":"path is \"/data/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.103064508Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"dataset is oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.109319316Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"snapshot is oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b@a24c7a39-7237-474e-ad7f-0f7e0e35068f","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.115080755Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b\", \"a24c7a39-7237-474e-ad7f-0f7e0e35068f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.115109386Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 94ab338b-919c-4baa-8f60-0b08f33fde3b-a24c7a39-7237-474e-ad7f-0f7e0e35068f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.123285731Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.123418083Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"uri":"/crucible/0/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b/snapshots/a24c7a39-7237-474e-ad7f-0f7e0e35068f/run","method":"POST","req_id":"80d1b8a1-004b-4442-8fa3-9a9e87551f18","remote_addr":"[fd00:1122:3344:101::b]:52324","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":20381,"response_code":200}
{"msg":"applying SMF actions for region 94ab338b-919c-4baa-8f60-0b08f33fde3b running snapshot a24c7a39-7237-474e-ad7f-0f7e0e35068f (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.123438873Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-94ab338b-919c-4baa-8f60-0b08f33fde3b-a24c7a39-7237-474e-ad7f-0f7e0e35068f","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.128186333Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-94ab338b-919c-4baa-8f60-0b08f33fde3b-a24c7a39-7237-474e-ad7f-0f7e0e35068f","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.145677034Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-94ab338b-919c-4baa-8f60-0b08f33fde3b-a24c7a39-7237-474e-ad7f-0f7e0e35068f","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.167151741Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b/.zfs/snapshot/a24c7a39-7237-474e-ad7f-0f7e0e35068f","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.167497917Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.167559389Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.167607279Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.16765433Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.167717001Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.174714064Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.183531801Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"region 94ab338b-919c-4baa-8f60-0b08f33fde3b running snapshot a24c7a39-7237-474e-ad7f-0f7e0e35068f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.183561901Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"path is \"/data/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.44661056Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"dataset is oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.452539843Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"snapshot is oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b@a24c7a39-7237-474e-ad7f-0f7e0e35068f","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.459094027Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b\", \"a24c7a39-7237-474e-ad7f-0f7e0e35068f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.459128537Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:07.465945166Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"uri":"/crucible/0/regions/94ab338b-919c-4baa-8f60-0b08f33fde3b/snapshots/a24c7a39-7237-474e-ad7f-0f7e0e35068f/run","method":"POST","req_id":"9a1e3b16-be9d-44d8-be82-76ef1cb0e08c","remote_addr":"[fd00:1122:3344:101::b]:52324","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":19441,"response_code":200}
{"msg":"region fcfe8209-39e9-4d83-a957-8f88275d1031 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.308789933Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.308950027Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.30965495Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"uri":"/crucible/0/regions","method":"POST","req_id":"965f3309-e5bf-4dfa-8f2d-9039ed1709f7","remote_addr":"[fd00:1122:3344:101::b]:52324","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":216,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible/regions/fcfe8209-39e9-4d83-a957-8f88275d1031","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.31549179Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible/regions/fcfe8209-39e9-4d83-a957-8f88275d1031","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.315538611Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"fcfe8209-39e9-4d83-a957-8f88275d1031\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/fcfe8209-39e9-4d83-a957-8f88275d1031\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.378982872Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"region":"fcfe8209-39e9-4d83-a957-8f88275d1031","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"fcfe8209-39e9-4d83-a957-8f88275d1031\" \"--data\" \"/data/regions/fcfe8209-39e9-4d83-a957-8f88275d1031\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.379026533Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"region":"fcfe8209-39e9-4d83-a957-8f88275d1031","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.473394449Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"uri":"/crucible/0/regions","method":"POST","req_id":"61b89473-aa3a-41b0-9ccb-caeea88d7919","remote_addr":"[fd00:1122:3344:101::b]:52324","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":63,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.595344948Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"region":"fcfe8209-39e9-4d83-a957-8f88275d1031","component":"worker"}
{"msg":"region fcfe8209-39e9-4d83-a957-8f88275d1031 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.595375368Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.59549066Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-fcfe8209-39e9-4d83-a957-8f88275d1031","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.597142891Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-fcfe8209-39e9-4d83-a957-8f88275d1031","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.601343981Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.616577759Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-fcfe8209-39e9-4d83-a957-8f88275d1031","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.624052571Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/fcfe8209-39e9-4d83-a957-8f88275d1031","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.624335006Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.624379347Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.624421098Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.624456119Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.63193641Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:14.641066513Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:15.132548186Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"uri":"/crucible/0/regions","method":"POST","req_id":"8c19c077-7b26-4b35-b964-56627c04040f","remote_addr":"[fd00:1122:3344:101::b]:52324","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":74,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:47.408324789Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:56708"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:47.408533603Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"uri":"/crucible/0/regions/fcfe8209-39e9-4d83-a957-8f88275d1031","method":"GET","req_id":"19e14ce2-915f-41ef-b83b-8f9bda3a6b72","remote_addr":"[fd00:1122:3344:101::b]:56708","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":61,"response_code":200}
{"msg":"path is \"/data/regions/fcfe8209-39e9-4d83-a957-8f88275d1031\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:47.411300385Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"dataset is oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible/regions/fcfe8209-39e9-4d83-a957-8f88275d1031","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:47.42267298Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"region fcfe8209-39e9-4d83-a957-8f88275d1031 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:47.429140763Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:47.429828276Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:47.429838066Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"uri":"/crucible/0/regions/fcfe8209-39e9-4d83-a957-8f88275d1031","method":"DELETE","req_id":"10e89897-2ba0-493a-a37d-d659c3d8775d","remote_addr":"[fd00:1122:3344:101::b]:56708","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":18547,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-fcfe8209-39e9-4d83-a957-8f88275d1031 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:47.430436047Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:47.43322011Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"uri":"/crucible/0/regions/fcfe8209-39e9-4d83-a957-8f88275d1031","method":"GET","req_id":"ee111d53-1f8c-4844-bcc7-a5b43fac0fad","remote_addr":"[fd00:1122:3344:101::b]:56708","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":36,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:47.445069794Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_35581ec7-b56d-4287-a4bd-df3b3177fd50/crucible/regions/fcfe8209-39e9-4d83-a957-8f88275d1031\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:47.453947172Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"region":"fcfe8209-39e9-4d83-a957-8f88275d1031","component":"worker"}
{"msg":"region fcfe8209-39e9-4d83-a957-8f88275d1031 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:47.502888909Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-11T22:28:47.572077798Z","hostname":"oxz_crucible_f7221c19-bcb6-4bac-ad0c-2c94d9b46dd8","pid":29656,"uri":"/crucible/0/regions/fcfe8209-39e9-4d83-a957-8f88275d1031","method":"GET","req_id":"eaa4bb51-87d5-461e-bbdc-3996d20d0d67","remote_addr":"[fd00:1122:3344:101::b]:56708","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":62,"response_code":200}