{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.439959881Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52412"} {"msg":"region 2759c2f3-98cd-45ac-a845-aec2ba7fef94 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.440233746Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.441280336Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"uri":"/crucible/0/regions","method":"POST","req_id":"7965c829-51cd-451c-bbb8-b7d730ab8614","remote_addr":"[fd00:1122:3344:101::a]:52412","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":1079,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.441293946Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_ff4de67a-e3c2-4d1d-8b26-14ae4a7394c1/crucible/regions/2759c2f3-98cd-45ac-a845-aec2ba7fef94","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.451493719Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_ff4de67a-e3c2-4d1d-8b26-14ae4a7394c1/crucible/regions/2759c2f3-98cd-45ac-a845-aec2ba7fef94","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.451512749Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"2759c2f3-98cd-45ac-a845-aec2ba7fef94\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2759c2f3-98cd-45ac-a845-aec2ba7fef94\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.516477467Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"region":"2759c2f3-98cd-45ac-a845-aec2ba7fef94","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2759c2f3-98cd-45ac-a845-aec2ba7fef94\" \"--data\" \"/data/regions/2759c2f3-98cd-45ac-a845-aec2ba7fef94\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.516506938Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"region":"2759c2f3-98cd-45ac-a845-aec2ba7fef94","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.591601538Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"region":"2759c2f3-98cd-45ac-a845-aec2ba7fef94","component":"worker"} {"msg":"region 2759c2f3-98cd-45ac-a845-aec2ba7fef94 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.591629768Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.59172369Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-2759c2f3-98cd-45ac-a845-aec2ba7fef94","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.592087297Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2759c2f3-98cd-45ac-a845-aec2ba7fef94","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.601332362Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.620352731Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2759c2f3-98cd-45ac-a845-aec2ba7fef94","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.623507871Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2759c2f3-98cd-45ac-a845-aec2ba7fef94","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.623793156Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.623839847Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.623877758Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.623912469Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.628288881Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.644883275Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:21.738898193Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"uri":"/crucible/0/regions","method":"POST","req_id":"d2005d27-2670-4e8e-9d46-87ec7789ed19","remote_addr":"[fd00:1122:3344:101::a]:52412","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":54,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.603237416Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"uri":"/crucible/0/regions/7c9983c4-f929-4011-8675-106aa0b4a5de","method":"GET","req_id":"2c52b32a-d2e8-48cc-83c2-470150002098","remote_addr":"[fd00:1122:3344:101::a]:52412","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":66,"response_code":200} {"msg":"path is \"/data/regions/7c9983c4-f929-4011-8675-106aa0b4a5de\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.606938335Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"datafile"} {"msg":"dataset is oxp_ff4de67a-e3c2-4d1d-8b26-14ae4a7394c1/crucible/regions/7c9983c4-f929-4011-8675-106aa0b4a5de","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.613101072Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"datafile"} {"msg":"snapshot is oxp_ff4de67a-e3c2-4d1d-8b26-14ae4a7394c1/crucible/regions/7c9983c4-f929-4011-8675-106aa0b4a5de@1daa339c-e17f-429e-82c2-817cb9caf564","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.618165778Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_ff4de67a-e3c2-4d1d-8b26-14ae4a7394c1/crucible/regions/7c9983c4-f929-4011-8675-106aa0b4a5de\", \"1daa339c-e17f-429e-82c2-817cb9caf564\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.618182278Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.625591018Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"uri":"/crucible/0/regions/7c9983c4-f929-4011-8675-106aa0b4a5de/snapshots/1daa339c-e17f-429e-82c2-817cb9caf564","method":"GET","req_id":"be87ca11-c7f6-4235-af72-eea9df867fbb","remote_addr":"[fd00:1122:3344:101::a]:52412","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":18691,"response_code":200} {"msg":"path is \"/data/regions/7c9983c4-f929-4011-8675-106aa0b4a5de\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.629263348Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"datafile"} {"msg":"dataset is oxp_ff4de67a-e3c2-4d1d-8b26-14ae4a7394c1/crucible/regions/7c9983c4-f929-4011-8675-106aa0b4a5de","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.636053006Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"datafile"} {"msg":"snapshot is oxp_ff4de67a-e3c2-4d1d-8b26-14ae4a7394c1/crucible/regions/7c9983c4-f929-4011-8675-106aa0b4a5de@1daa339c-e17f-429e-82c2-817cb9caf564","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.642843684Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_ff4de67a-e3c2-4d1d-8b26-14ae4a7394c1/crucible/regions/7c9983c4-f929-4011-8675-106aa0b4a5de\", \"1daa339c-e17f-429e-82c2-817cb9caf564\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.642863425Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 7c9983c4-f929-4011-8675-106aa0b4a5de-1daa339c-e17f-429e-82c2-817cb9caf564 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.651730682Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.651864645Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"uri":"/crucible/0/regions/7c9983c4-f929-4011-8675-106aa0b4a5de/snapshots/1daa339c-e17f-429e-82c2-817cb9caf564/run","method":"POST","req_id":"10512a5c-959e-43fa-a553-0ec23a69e7c9","remote_addr":"[fd00:1122:3344:101::a]:52412","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":22625,"response_code":200} {"msg":"applying SMF actions for region 7c9983c4-f929-4011-8675-106aa0b4a5de running snapshot 1daa339c-e17f-429e-82c2-817cb9caf564 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.651883185Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-7c9983c4-f929-4011-8675-106aa0b4a5de-1daa339c-e17f-429e-82c2-817cb9caf564","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.656431751Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-7c9983c4-f929-4011-8675-106aa0b4a5de-1daa339c-e17f-429e-82c2-817cb9caf564","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.665709807Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-7c9983c4-f929-4011-8675-106aa0b4a5de-1daa339c-e17f-429e-82c2-817cb9caf564","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.675678095Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7c9983c4-f929-4011-8675-106aa0b4a5de/.zfs/snapshot/1daa339c-e17f-429e-82c2-817cb9caf564","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.67592096Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.675972491Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.676008481Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.676045302Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.676091293Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.679794843Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.684363939Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"worker"} {"msg":"region 7c9983c4-f929-4011-8675-106aa0b4a5de running snapshot 1daa339c-e17f-429e-82c2-817cb9caf564 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.68437422Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"datafile"} {"msg":"path is \"/data/regions/7c9983c4-f929-4011-8675-106aa0b4a5de\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.942063332Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"datafile"} {"msg":"dataset is oxp_ff4de67a-e3c2-4d1d-8b26-14ae4a7394c1/crucible/regions/7c9983c4-f929-4011-8675-106aa0b4a5de","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.948740029Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"datafile"} {"msg":"snapshot is oxp_ff4de67a-e3c2-4d1d-8b26-14ae4a7394c1/crucible/regions/7c9983c4-f929-4011-8675-106aa0b4a5de@1daa339c-e17f-429e-82c2-817cb9caf564","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.955127179Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_ff4de67a-e3c2-4d1d-8b26-14ae4a7394c1/crucible/regions/7c9983c4-f929-4011-8675-106aa0b4a5de\", \"1daa339c-e17f-429e-82c2-817cb9caf564\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.95513948Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-01T20:24:23.961262366Z","hostname":"oxz_crucible_0730b941-4d71-4fd3-972e-c465c035e5f1","pid":24789,"uri":"/crucible/0/regions/7c9983c4-f929-4011-8675-106aa0b4a5de/snapshots/1daa339c-e17f-429e-82c2-817cb9caf564/run","method":"POST","req_id":"2461bd4e-17ea-4cb5-8edd-e9db32512590","remote_addr":"[fd00:1122:3344:101::a]:52412","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":19239,"response_code":200}