{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:09.886555083Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:55533"} {"msg":"region e629aa9e-464b-4f89-b742-ab165c5455b5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:09.909499561Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:09.91229163Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"uri":"/crucible/0/regions","method":"POST","req_id":"296d0ba4-120f-422a-8c3f-7092162d6186","remote_addr":"[fd00:1122:3344:101::c]:55533","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":14763,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:09.914671457Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_5699e9aa-cc1a-421f-9db9-537af91db6b9/crucible/regions/e629aa9e-464b-4f89-b742-ab165c5455b5","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:09.924870985Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_5699e9aa-cc1a-421f-9db9-537af91db6b9/crucible/regions/e629aa9e-464b-4f89-b742-ab165c5455b5","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:09.924903046Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"e629aa9e-464b-4f89-b742-ab165c5455b5\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e629aa9e-464b-4f89-b742-ab165c5455b5\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:09.969504276Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"region":"e629aa9e-464b-4f89-b742-ab165c5455b5","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e629aa9e-464b-4f89-b742-ab165c5455b5\" \"--data\" \"/data/regions/e629aa9e-464b-4f89-b742-ab165c5455b5\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:09.969554908Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"region":"e629aa9e-464b-4f89-b742-ab165c5455b5","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:10.218654024Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"uri":"/crucible/0/regions","method":"POST","req_id":"317eae81-4f1d-455b-b9da-77c3002c919c","remote_addr":"[fd00:1122:3344:101::c]:55533","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":69,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:10.278474544Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"region":"e629aa9e-464b-4f89-b742-ab165c5455b5","component":"worker"} {"msg":"region e629aa9e-464b-4f89-b742-ab165c5455b5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:10.278518825Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:10.278660969Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-e629aa9e-464b-4f89-b742-ab165c5455b5","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:10.279128082Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e629aa9e-464b-4f89-b742-ab165c5455b5","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:10.283482895Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:10.295833924Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e629aa9e-464b-4f89-b742-ab165c5455b5","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:10.300496576Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e629aa9e-464b-4f89-b742-ab165c5455b5","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:10.30133492Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:10.301403322Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:10.301456223Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:10.301489264Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:10.305901939Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:10.310068896Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:45:10.705155576Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"uri":"/crucible/0/regions","method":"POST","req_id":"3fc8fedd-7d10-4c23-a6f4-f15b7f8b8d00","remote_addr":"[fd00:1122:3344:101::c]:55533","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":58,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.399893256Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:54076"} {"msg":"region 1d169c69-8b64-4305-ad9b-ec970d1c3e34 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.400531644Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.400611936Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"uri":"/crucible/0/regions","method":"POST","req_id":"2cd056f8-9d2f-4d5d-ab09-c2838dad02c6","remote_addr":"[fd00:1122:3344:101::c]:54076","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":668,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.400629757Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_5699e9aa-cc1a-421f-9db9-537af91db6b9/crucible/regions/1d169c69-8b64-4305-ad9b-ec970d1c3e34","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.413212932Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_5699e9aa-cc1a-421f-9db9-537af91db6b9/crucible/regions/1d169c69-8b64-4305-ad9b-ec970d1c3e34","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.413251823Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"1d169c69-8b64-4305-ad9b-ec970d1c3e34\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1d169c69-8b64-4305-ad9b-ec970d1c3e34\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.461200498Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"region":"1d169c69-8b64-4305-ad9b-ec970d1c3e34","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1d169c69-8b64-4305-ad9b-ec970d1c3e34\" \"--data\" \"/data/regions/1d169c69-8b64-4305-ad9b-ec970d1c3e34\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.46126093Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"region":"1d169c69-8b64-4305-ad9b-ec970d1c3e34","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.547733782Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"region":"1d169c69-8b64-4305-ad9b-ec970d1c3e34","component":"worker"} {"msg":"region 1d169c69-8b64-4305-ad9b-ec970d1c3e34 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.547776763Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.547874986Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-1d169c69-8b64-4305-ad9b-ec970d1c3e34","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.54835489Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1d169c69-8b64-4305-ad9b-ec970d1c3e34","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.559038572Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.581940019Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1d169c69-8b64-4305-ad9b-ec970d1c3e34","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.586366824Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1d169c69-8b64-4305-ad9b-ec970d1c3e34","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.586661452Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.586716434Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.586764785Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.586797936Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.59085685Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.596653104Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:50.767023297Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"uri":"/crucible/0/regions","method":"POST","req_id":"cab99fc6-87b7-412c-aa12-1cf06b40b5d3","remote_addr":"[fd00:1122:3344:101::c]:54076","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":58,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.277388081Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"uri":"/crucible/0/regions/e629aa9e-464b-4f89-b742-ab165c5455b5","method":"GET","req_id":"a739ed9c-45e1-46d5-b645-51ee16a0791a","remote_addr":"[fd00:1122:3344:101::c]:54076","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":2974,"response_code":"200"} {"msg":"path is \"/data/regions/e629aa9e-464b-4f89-b742-ab165c5455b5\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.281157608Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"datafile"} {"msg":"dataset is oxp_5699e9aa-cc1a-421f-9db9-537af91db6b9/crucible/regions/e629aa9e-464b-4f89-b742-ab165c5455b5","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.290718248Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"datafile"} {"msg":"snapshot is oxp_5699e9aa-cc1a-421f-9db9-537af91db6b9/crucible/regions/e629aa9e-464b-4f89-b742-ab165c5455b5@267b70a6-9a72-4846-a7be-73cc1065cb5b","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.300739171Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_5699e9aa-cc1a-421f-9db9-537af91db6b9/crucible/regions/e629aa9e-464b-4f89-b742-ab165c5455b5\", \"267b70a6-9a72-4846-a7be-73cc1065cb5b\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.300770002Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.312353129Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"uri":"/crucible/0/regions/e629aa9e-464b-4f89-b742-ab165c5455b5/snapshots/267b70a6-9a72-4846-a7be-73cc1065cb5b","method":"GET","req_id":"eaf87100-0d05-455c-b16d-38305588b3d5","remote_addr":"[fd00:1122:3344:101::c]:54076","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":31199,"response_code":"200"} {"msg":"path is \"/data/regions/e629aa9e-464b-4f89-b742-ab165c5455b5\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.315665522Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"datafile"} {"msg":"dataset is oxp_5699e9aa-cc1a-421f-9db9-537af91db6b9/crucible/regions/e629aa9e-464b-4f89-b742-ab165c5455b5","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.324361118Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"datafile"} {"msg":"snapshot is oxp_5699e9aa-cc1a-421f-9db9-537af91db6b9/crucible/regions/e629aa9e-464b-4f89-b742-ab165c5455b5@267b70a6-9a72-4846-a7be-73cc1065cb5b","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.332298232Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_5699e9aa-cc1a-421f-9db9-537af91db6b9/crucible/regions/e629aa9e-464b-4f89-b742-ab165c5455b5\", \"267b70a6-9a72-4846-a7be-73cc1065cb5b\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.332333013Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot e629aa9e-464b-4f89-b742-ab165c5455b5-267b70a6-9a72-4846-a7be-73cc1065cb5b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.342166711Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"datafile"} {"msg":"applying SMF actions for region e629aa9e-464b-4f89-b742-ab165c5455b5 running snapshot 267b70a6-9a72-4846-a7be-73cc1065cb5b (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.342362927Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.342377697Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"uri":"/crucible/0/regions/e629aa9e-464b-4f89-b742-ab165c5455b5/snapshots/267b70a6-9a72-4846-a7be-73cc1065cb5b/run","method":"POST","req_id":"ee4519f3-097d-4558-8dc2-3f23a26a1512","remote_addr":"[fd00:1122:3344:101::c]:54076","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":26741,"response_code":"200"} {"msg":"creating missing snapshot instance snapshot-e629aa9e-464b-4f89-b742-ab165c5455b5-267b70a6-9a72-4846-a7be-73cc1065cb5b","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.344568859Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-e629aa9e-464b-4f89-b742-ab165c5455b5-267b70a6-9a72-4846-a7be-73cc1065cb5b","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.356813355Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-e629aa9e-464b-4f89-b742-ab165c5455b5-267b70a6-9a72-4846-a7be-73cc1065cb5b","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.366255042Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e629aa9e-464b-4f89-b742-ab165c5455b5/.zfs/snapshot/267b70a6-9a72-4846-a7be-73cc1065cb5b","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.366519339Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.3665716Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.366613212Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.366654913Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.366710714Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.370570693Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.375658457Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"region e629aa9e-464b-4f89-b742-ab165c5455b5 running snapshot 267b70a6-9a72-4846-a7be-73cc1065cb5b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.375682308Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"datafile"} {"msg":"path is \"/data/regions/e629aa9e-464b-4f89-b742-ab165c5455b5\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.550125845Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"datafile"} {"msg":"dataset is oxp_5699e9aa-cc1a-421f-9db9-537af91db6b9/crucible/regions/e629aa9e-464b-4f89-b742-ab165c5455b5","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.558888803Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"datafile"} {"msg":"snapshot is oxp_5699e9aa-cc1a-421f-9db9-537af91db6b9/crucible/regions/e629aa9e-464b-4f89-b742-ab165c5455b5@267b70a6-9a72-4846-a7be-73cc1065cb5b","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.566946911Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_5699e9aa-cc1a-421f-9db9-537af91db6b9/crucible/regions/e629aa9e-464b-4f89-b742-ab165c5455b5\", \"267b70a6-9a72-4846-a7be-73cc1065cb5b\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.566980401Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:52.575301356Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"uri":"/crucible/0/regions/e629aa9e-464b-4f89-b742-ab165c5455b5/snapshots/267b70a6-9a72-4846-a7be-73cc1065cb5b/run","method":"POST","req_id":"0730a10e-8d96-4cc0-ae13-aed02985cb2e","remote_addr":"[fd00:1122:3344:101::c]:54076","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":25195,"response_code":"200"} {"msg":"region e578e877-4a32-409c-aeff-9a4218e6949b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:56.948434907Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:56.948591862Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"uri":"/crucible/0/regions","method":"POST","req_id":"b6a3b806-f31a-41c7-9dbc-214a115b4f66","remote_addr":"[fd00:1122:3344:101::c]:54076","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":214,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:56.948620403Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_5699e9aa-cc1a-421f-9db9-537af91db6b9/crucible/regions/e578e877-4a32-409c-aeff-9a4218e6949b","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:56.957786122Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_5699e9aa-cc1a-421f-9db9-537af91db6b9/crucible/regions/e578e877-4a32-409c-aeff-9a4218e6949b","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:56.957816422Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"e578e877-4a32-409c-aeff-9a4218e6949b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e578e877-4a32-409c-aeff-9a4218e6949b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:57.003007799Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"region":"e578e877-4a32-409c-aeff-9a4218e6949b","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e578e877-4a32-409c-aeff-9a4218e6949b\" \"--data\" \"/data/regions/e578e877-4a32-409c-aeff-9a4218e6949b\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:57.003066201Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"region":"e578e877-4a32-409c-aeff-9a4218e6949b","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:57.123630556Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"region":"e578e877-4a32-409c-aeff-9a4218e6949b","component":"worker"} {"msg":"region e578e877-4a32-409c-aeff-9a4218e6949b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:57.123677688Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:57.123782051Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-e578e877-4a32-409c-aeff-9a4218e6949b","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:57.125049616Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e578e877-4a32-409c-aeff-9a4218e6949b","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:57.128440492Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:57.136067698Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e578e877-4a32-409c-aeff-9a4218e6949b","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:57.139149725Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e578e877-4a32-409c-aeff-9a4218e6949b","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:57.139428653Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:57.139489794Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:57.139568617Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:57.139626338Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:57.143080396Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:57.154383585Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:55:57.241357601Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"uri":"/crucible/0/regions","method":"POST","req_id":"4bd6cd6d-5712-4916-9740-2b0bba19dd3e","remote_addr":"[fd00:1122:3344:101::c]:54076","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":73,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:56:22.501929275Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"uri":"/crucible/0/regions/e578e877-4a32-409c-aeff-9a4218e6949b","method":"GET","req_id":"5d5761c2-5f0d-4f76-82c3-66f32e029fad","remote_addr":"[fd00:1122:3344:101::c]:54076","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":47,"response_code":"200"} {"msg":"path is \"/data/regions/e578e877-4a32-409c-aeff-9a4218e6949b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:56:22.504496698Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"datafile"} {"msg":"dataset is oxp_5699e9aa-cc1a-421f-9db9-537af91db6b9/crucible/regions/e578e877-4a32-409c-aeff-9a4218e6949b","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:56:22.519240934Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"datafile"} {"msg":"region e578e877-4a32-409c-aeff-9a4218e6949b state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:56:22.52900794Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:56:22.529192865Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-e578e877-4a32-409c-aeff-9a4218e6949b (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:56:22.53042523Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:56:22.531403108Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"uri":"/crucible/0/regions/e578e877-4a32-409c-aeff-9a4218e6949b","method":"DELETE","req_id":"0bd7119b-2ea9-4600-b023-93939729f14a","remote_addr":"[fd00:1122:3344:101::c]:54076","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":26933,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:56:22.534253538Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"uri":"/crucible/0/regions/e578e877-4a32-409c-aeff-9a4218e6949b","method":"GET","req_id":"d508a585-4570-4b35-bfd1-438187f94b64","remote_addr":"[fd00:1122:3344:101::c]:54076","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":27,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:56:22.543598822Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_5699e9aa-cc1a-421f-9db9-537af91db6b9/crucible/regions/e578e877-4a32-409c-aeff-9a4218e6949b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:56:22.554951583Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"region":"e578e877-4a32-409c-aeff-9a4218e6949b","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:56:22.595304243Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"uri":"/crucible/0/regions/e578e877-4a32-409c-aeff-9a4218e6949b","method":"GET","req_id":"7a53e817-8b68-4b68-92d4-d8c278834ef9","remote_addr":"[fd00:1122:3344:101::c]:54076","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":83,"response_code":"200"} {"msg":"region e578e877-4a32-409c-aeff-9a4218e6949b state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:56:22.624314362Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-19T11:56:22.692351334Z","hostname":"oxz_crucible_5e7f259c-ca8f-4e85-815a-65519ada1762","pid":23530,"uri":"/crucible/0/regions/e578e877-4a32-409c-aeff-9a4218e6949b","method":"GET","req_id":"c800d919-4f6e-45a9-bd19-fd5c1a83efa6","remote_addr":"[fd00:1122:3344:101::c]:54076","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":32,"response_code":"200"}