{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:08.937257725Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:50846"} {"msg":"region bf9903a6-0553-42a0-b573-167d1f921259 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:08.937474266Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:08.939722666Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"uri":"/crucible/0/regions","method":"POST","req_id":"0d706401-bf42-4dd9-9916-8fd2ff66be78","remote_addr":"[fd00:1122:3344:101::a]:50846","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":2292,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:08.939734115Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_5d653a63-c398-4932-9454-a6f807060158/crucible/regions/bf9903a6-0553-42a0-b573-167d1f921259","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:08.951868158Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_5d653a63-c398-4932-9454-a6f807060158/crucible/regions/bf9903a6-0553-42a0-b573-167d1f921259","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:08.951888318Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:09.072672082Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"uri":"/crucible/0/regions","method":"POST","req_id":"4e1c6c9e-334f-4ad9-83aa-47a6d423eef9","remote_addr":"[fd00:1122:3344:101::a]:50846","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":44,"response_code":200} {"msg":"creating region Region { id: RegionId(\"bf9903a6-0553-42a0-b573-167d1f921259\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/bf9903a6-0553-42a0-b573-167d1f921259\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:09.601318526Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"region":"bf9903a6-0553-42a0-b573-167d1f921259","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"bf9903a6-0553-42a0-b573-167d1f921259\" \"--data\" \"/data/regions/bf9903a6-0553-42a0-b573-167d1f921259\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:09.601355176Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"region":"bf9903a6-0553-42a0-b573-167d1f921259","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:09.627303279Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"uri":"/crucible/0/regions","method":"POST","req_id":"f953985e-6b04-4684-a09a-8c3b3ad5c532","remote_addr":"[fd00:1122:3344:101::a]:50846","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":44,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:09.747008538Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"region":"bf9903a6-0553-42a0-b573-167d1f921259","component":"worker"} {"msg":"region bf9903a6-0553-42a0-b573-167d1f921259 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:09.747034748Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:09.747150129Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-bf9903a6-0553-42a0-b573-167d1f921259","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:09.767079879Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-bf9903a6-0553-42a0-b573-167d1f921259","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:09.774185746Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:09.788301377Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bf9903a6-0553-42a0-b573-167d1f921259","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:09.791781802Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bf9903a6-0553-42a0-b573-167d1f921259","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:09.792070244Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:09.792120634Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:09.792161614Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:09.792202204Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:09.795478868Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:09.801500713Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:10.288165991Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"uri":"/crucible/0/regions","method":"POST","req_id":"404ebca0-6778-46dc-b0d3-d4b474246a0e","remote_addr":"[fd00:1122:3344:101::a]:50846","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":59,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.380389643Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"uri":"/crucible/0/regions/65d7e0c2-43f9-4b01-b02e-7d131a05dee6","method":"GET","req_id":"06138d9d-07e2-4050-946b-e20b339a8b0f","remote_addr":"[fd00:1122:3344:101::a]:50846","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":74,"response_code":200} {"msg":"path is \"/data/regions/65d7e0c2-43f9-4b01-b02e-7d131a05dee6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.383389586Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"datafile"} {"msg":"dataset is oxp_5d653a63-c398-4932-9454-a6f807060158/crucible/regions/65d7e0c2-43f9-4b01-b02e-7d131a05dee6","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.389939075Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"datafile"} {"msg":"snapshot is oxp_5d653a63-c398-4932-9454-a6f807060158/crucible/regions/65d7e0c2-43f9-4b01-b02e-7d131a05dee6@67819a7a-9ac6-41e3-9914-01e877dd2567","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.39575346Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_5d653a63-c398-4932-9454-a6f807060158/crucible/regions/65d7e0c2-43f9-4b01-b02e-7d131a05dee6\", \"67819a7a-9ac6-41e3-9914-01e877dd2567\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.39577578Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.405174951Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"uri":"/crucible/0/regions/65d7e0c2-43f9-4b01-b02e-7d131a05dee6/snapshots/67819a7a-9ac6-41e3-9914-01e877dd2567","method":"GET","req_id":"b4426ef0-5977-4a50-8c2a-f97f28c05921","remote_addr":"[fd00:1122:3344:101::a]:50846","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":21807,"response_code":200} {"msg":"path is \"/data/regions/65d7e0c2-43f9-4b01-b02e-7d131a05dee6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.408164274Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"datafile"} {"msg":"dataset is oxp_5d653a63-c398-4932-9454-a6f807060158/crucible/regions/65d7e0c2-43f9-4b01-b02e-7d131a05dee6","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.414646782Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"datafile"} {"msg":"snapshot is oxp_5d653a63-c398-4932-9454-a6f807060158/crucible/regions/65d7e0c2-43f9-4b01-b02e-7d131a05dee6@67819a7a-9ac6-41e3-9914-01e877dd2567","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.420514497Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_5d653a63-c398-4932-9454-a6f807060158/crucible/regions/65d7e0c2-43f9-4b01-b02e-7d131a05dee6\", \"67819a7a-9ac6-41e3-9914-01e877dd2567\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.420560568Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 65d7e0c2-43f9-4b01-b02e-7d131a05dee6-67819a7a-9ac6-41e3-9914-01e877dd2567 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.427700879Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.427853009Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"uri":"/crucible/0/regions/65d7e0c2-43f9-4b01-b02e-7d131a05dee6/snapshots/67819a7a-9ac6-41e3-9914-01e877dd2567/run","method":"POST","req_id":"4507df48-7887-41d1-9e02-350cb74b15ab","remote_addr":"[fd00:1122:3344:101::a]:50846","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":19711,"response_code":200} {"msg":"applying SMF actions for region 65d7e0c2-43f9-4b01-b02e-7d131a05dee6 running snapshot 67819a7a-9ac6-41e3-9914-01e877dd2567 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.427868899Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-65d7e0c2-43f9-4b01-b02e-7d131a05dee6-67819a7a-9ac6-41e3-9914-01e877dd2567","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.432329189Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-65d7e0c2-43f9-4b01-b02e-7d131a05dee6-67819a7a-9ac6-41e3-9914-01e877dd2567","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.438397105Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-65d7e0c2-43f9-4b01-b02e-7d131a05dee6-67819a7a-9ac6-41e3-9914-01e877dd2567","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.448659999Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/65d7e0c2-43f9-4b01-b02e-7d131a05dee6/.zfs/snapshot/67819a7a-9ac6-41e3-9914-01e877dd2567","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.4488502Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.44889182Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.448927701Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.448961531Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.449003901Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.452485796Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.456653514Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"worker"} {"msg":"region 65d7e0c2-43f9-4b01-b02e-7d131a05dee6 running snapshot 67819a7a-9ac6-41e3-9914-01e877dd2567 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.456669544Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"datafile"} {"msg":"path is \"/data/regions/65d7e0c2-43f9-4b01-b02e-7d131a05dee6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.628321069Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"datafile"} {"msg":"dataset is oxp_5d653a63-c398-4932-9454-a6f807060158/crucible/regions/65d7e0c2-43f9-4b01-b02e-7d131a05dee6","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.634468686Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"datafile"} {"msg":"snapshot is oxp_5d653a63-c398-4932-9454-a6f807060158/crucible/regions/65d7e0c2-43f9-4b01-b02e-7d131a05dee6@67819a7a-9ac6-41e3-9914-01e877dd2567","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.640835203Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_5d653a63-c398-4932-9454-a6f807060158/crucible/regions/65d7e0c2-43f9-4b01-b02e-7d131a05dee6\", \"67819a7a-9ac6-41e3-9914-01e877dd2567\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.640879533Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-19T20:15:11.646336937Z","hostname":"oxz_crucible_a1261582-52f0-41df-a791-552d62b81c42","pid":25524,"uri":"/crucible/0/regions/65d7e0c2-43f9-4b01-b02e-7d131a05dee6/snapshots/67819a7a-9ac6-41e3-9914-01e877dd2567/run","method":"POST","req_id":"5fedcaa7-2bc8-4468-93f7-4e7858e144b5","remote_addr":"[fd00:1122:3344:101::a]:50846","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":18057,"response_code":200}