{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.110345574Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:54423"} {"msg":"region 2059680f-4d3d-4909-9ec4-05bccaf0330e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.110492328Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.111117744Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"uri":"/crucible/0/regions","method":"POST","req_id":"296b8079-b07b-41e1-a5ae-4c8610619589","remote_addr":"[fd00:1122:3344:101::b]:54423","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":669,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.111133374Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_e61cdce6-2f7f-4a42-afec-d6b28150452c/crucible/regions/2059680f-4d3d-4909-9ec4-05bccaf0330e","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.119022386Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_e61cdce6-2f7f-4a42-afec-d6b28150452c/crucible/regions/2059680f-4d3d-4909-9ec4-05bccaf0330e","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.119040696Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"2059680f-4d3d-4909-9ec4-05bccaf0330e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2059680f-4d3d-4909-9ec4-05bccaf0330e\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.199577733Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"region":"2059680f-4d3d-4909-9ec4-05bccaf0330e","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2059680f-4d3d-4909-9ec4-05bccaf0330e\" \"--data\" \"/data/regions/2059680f-4d3d-4909-9ec4-05bccaf0330e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.199608624Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"region":"2059680f-4d3d-4909-9ec4-05bccaf0330e","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.340555404Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"uri":"/crucible/0/regions","method":"POST","req_id":"72620f4c-ae16-438b-b2b3-f092e6f67da5","remote_addr":"[fd00:1122:3344:101::b]:54423","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":60,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.366699762Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"region":"2059680f-4d3d-4909-9ec4-05bccaf0330e","component":"worker"} {"msg":"region 2059680f-4d3d-4909-9ec4-05bccaf0330e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.366717922Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.366856646Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-2059680f-4d3d-4909-9ec4-05bccaf0330e","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.36779059Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2059680f-4d3d-4909-9ec4-05bccaf0330e","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.400717681Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.430226635Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2059680f-4d3d-4909-9ec4-05bccaf0330e","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.441094662Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2059680f-4d3d-4909-9ec4-05bccaf0330e","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.44139393Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.441454311Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.441496143Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.441533123Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.446164152Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.479822431Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:12.888697086Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"uri":"/crucible/0/regions","method":"POST","req_id":"9bc409c2-06df-4804-8454-d6f731d9bfd0","remote_addr":"[fd00:1122:3344:101::b]:54423","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":63,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.148966217Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"uri":"/crucible/0/regions/cab83ceb-4dc3-403d-bdd8-ed555487cd33","method":"GET","req_id":"915d6ecd-6989-4ea1-97cc-0b7e886db9fa","remote_addr":"[fd00:1122:3344:101::b]:54423","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":71,"response_code":200} {"msg":"path is \"/data/regions/cab83ceb-4dc3-403d-bdd8-ed555487cd33\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.152746463Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"datafile"} {"msg":"dataset is oxp_e61cdce6-2f7f-4a42-afec-d6b28150452c/crucible/regions/cab83ceb-4dc3-403d-bdd8-ed555487cd33","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.158711106Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"datafile"} {"msg":"snapshot is oxp_e61cdce6-2f7f-4a42-afec-d6b28150452c/crucible/regions/cab83ceb-4dc3-403d-bdd8-ed555487cd33@daf70307-7db2-49dc-ada4-0c7ea02b542f","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.165439038Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_e61cdce6-2f7f-4a42-afec-d6b28150452c/crucible/regions/cab83ceb-4dc3-403d-bdd8-ed555487cd33\", \"daf70307-7db2-49dc-ada4-0c7ea02b542f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.165460118Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.174705584Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"uri":"/crucible/0/regions/cab83ceb-4dc3-403d-bdd8-ed555487cd33/snapshots/daf70307-7db2-49dc-ada4-0c7ea02b542f","method":"GET","req_id":"0bbc1821-05b5-4d9f-a863-9623ff249e28","remote_addr":"[fd00:1122:3344:101::b]:54423","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":21973,"response_code":200} {"msg":"path is \"/data/regions/cab83ceb-4dc3-403d-bdd8-ed555487cd33\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.178287176Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"datafile"} {"msg":"dataset is oxp_e61cdce6-2f7f-4a42-afec-d6b28150452c/crucible/regions/cab83ceb-4dc3-403d-bdd8-ed555487cd33","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.18471142Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"datafile"} {"msg":"snapshot is oxp_e61cdce6-2f7f-4a42-afec-d6b28150452c/crucible/regions/cab83ceb-4dc3-403d-bdd8-ed555487cd33@daf70307-7db2-49dc-ada4-0c7ea02b542f","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.190830476Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_e61cdce6-2f7f-4a42-afec-d6b28150452c/crucible/regions/cab83ceb-4dc3-403d-bdd8-ed555487cd33\", \"daf70307-7db2-49dc-ada4-0c7ea02b542f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.190856007Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot cab83ceb-4dc3-403d-bdd8-ed555487cd33-daf70307-7db2-49dc-ada4-0c7ea02b542f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.203093359Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.203235323Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"uri":"/crucible/0/regions/cab83ceb-4dc3-403d-bdd8-ed555487cd33/snapshots/daf70307-7db2-49dc-ada4-0c7ea02b542f/run","method":"POST","req_id":"e49b9602-55a5-410b-8468-4cc2d13701b1","remote_addr":"[fd00:1122:3344:101::b]:54423","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":24970,"response_code":200} {"msg":"applying SMF actions for region cab83ceb-4dc3-403d-bdd8-ed555487cd33 running snapshot daf70307-7db2-49dc-ada4-0c7ea02b542f (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.203252203Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-cab83ceb-4dc3-403d-bdd8-ed555487cd33-daf70307-7db2-49dc-ada4-0c7ea02b542f","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.208434936Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-cab83ceb-4dc3-403d-bdd8-ed555487cd33-daf70307-7db2-49dc-ada4-0c7ea02b542f","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.220213117Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-cab83ceb-4dc3-403d-bdd8-ed555487cd33-daf70307-7db2-49dc-ada4-0c7ea02b542f","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.245135113Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cab83ceb-4dc3-403d-bdd8-ed555487cd33/.zfs/snapshot/daf70307-7db2-49dc-ada4-0c7ea02b542f","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.24539898Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.245442661Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.245482062Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.245517623Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.245562014Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.254525443Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.260070445Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"worker"} {"msg":"region cab83ceb-4dc3-403d-bdd8-ed555487cd33 running snapshot daf70307-7db2-49dc-ada4-0c7ea02b542f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.260080575Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"datafile"} {"msg":"path is \"/data/regions/cab83ceb-4dc3-403d-bdd8-ed555487cd33\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.334158107Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"datafile"} {"msg":"dataset is oxp_e61cdce6-2f7f-4a42-afec-d6b28150452c/crucible/regions/cab83ceb-4dc3-403d-bdd8-ed555487cd33","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.340115619Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"datafile"} {"msg":"snapshot is oxp_e61cdce6-2f7f-4a42-afec-d6b28150452c/crucible/regions/cab83ceb-4dc3-403d-bdd8-ed555487cd33@daf70307-7db2-49dc-ada4-0c7ea02b542f","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.34523471Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_e61cdce6-2f7f-4a42-afec-d6b28150452c/crucible/regions/cab83ceb-4dc3-403d-bdd8-ed555487cd33\", \"daf70307-7db2-49dc-ada4-0c7ea02b542f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.34524956Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:10:14.351421998Z","hostname":"oxz_crucible_2c33cfba-caaf-45b1-be57-b3d199c188b2","pid":25805,"uri":"/crucible/0/regions/cab83ceb-4dc3-403d-bdd8-ed555487cd33/snapshots/daf70307-7db2-49dc-ada4-0c7ea02b542f/run","method":"POST","req_id":"fadc07f8-e506-45cc-bc79-e377aec6d140","remote_addr":"[fd00:1122:3344:101::b]:54423","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":17299,"response_code":200}