{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.753464277Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:48256"} {"msg":"region a1e40e3e-1361-48d4-843d-9d0e79ed28ca state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.753559328Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.754224246Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.754254156Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"uri":"/crucible/0/regions","method":"POST","req_id":"5079e562-a074-4cb0-a68b-95aea61142d4","remote_addr":"[fd00:1122:3344:101::a]:48256","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":724,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_3e944a96-ab1b-47aa-a595-7e8c9f9e5446/crucible/regions/a1e40e3e-1361-48d4-843d-9d0e79ed28ca","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.764049041Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_3e944a96-ab1b-47aa-a595-7e8c9f9e5446/crucible/regions/a1e40e3e-1361-48d4-843d-9d0e79ed28ca","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.764079182Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"a1e40e3e-1361-48d4-843d-9d0e79ed28ca\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a1e40e3e-1361-48d4-843d-9d0e79ed28ca\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.809734629Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"region":"a1e40e3e-1361-48d4-843d-9d0e79ed28ca","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a1e40e3e-1361-48d4-843d-9d0e79ed28ca\" \"--data\" \"/data/regions/a1e40e3e-1361-48d4-843d-9d0e79ed28ca\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.809778969Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"region":"a1e40e3e-1361-48d4-843d-9d0e79ed28ca","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.881381942Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"region":"a1e40e3e-1361-48d4-843d-9d0e79ed28ca","component":"worker"} {"msg":"region a1e40e3e-1361-48d4-843d-9d0e79ed28ca state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.881424182Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.881531883Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-a1e40e3e-1361-48d4-843d-9d0e79ed28ca","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.894279843Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a1e40e3e-1361-48d4-843d-9d0e79ed28ca","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.898162129Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.906842031Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a1e40e3e-1361-48d4-843d-9d0e79ed28ca","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.90842176Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a1e40e3e-1361-48d4-843d-9d0e79ed28ca","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.908704953Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.908770384Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.908825054Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.908875595Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.911278983Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:19.915014307Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:20.057186189Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"uri":"/crucible/0/regions","method":"POST","req_id":"419d7324-4dfe-4091-9b76-ddffec6e5a7a","remote_addr":"[fd00:1122:3344:101::a]:48256","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":89,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.136588775Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"uri":"/crucible/0/regions/0b612c0b-3943-42cc-8df5-b98b8aa1c167","method":"GET","req_id":"3c97a46c-289d-4e98-b3d1-5193f7fca6b0","remote_addr":"[fd00:1122:3344:101::a]:48256","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":85,"response_code":200} {"msg":"path is \"/data/regions/0b612c0b-3943-42cc-8df5-b98b8aa1c167\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.142599796Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"datafile"} {"msg":"dataset is oxp_3e944a96-ab1b-47aa-a595-7e8c9f9e5446/crucible/regions/0b612c0b-3943-42cc-8df5-b98b8aa1c167","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.149886782Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"datafile"} {"msg":"snapshot is oxp_3e944a96-ab1b-47aa-a595-7e8c9f9e5446/crucible/regions/0b612c0b-3943-42cc-8df5-b98b8aa1c167@f525cb9e-65ff-4e02-943f-e77f4719895c","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.156618451Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_3e944a96-ab1b-47aa-a595-7e8c9f9e5446/crucible/regions/0b612c0b-3943-42cc-8df5-b98b8aa1c167\", \"f525cb9e-65ff-4e02-943f-e77f4719895c\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.156634411Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.164597155Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"uri":"/crucible/0/regions/0b612c0b-3943-42cc-8df5-b98b8aa1c167/snapshots/f525cb9e-65ff-4e02-943f-e77f4719895c","method":"GET","req_id":"ba7729ef-225c-42e9-8ece-301c49098c5c","remote_addr":"[fd00:1122:3344:101::a]:48256","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":22034,"response_code":200} {"msg":"path is \"/data/regions/0b612c0b-3943-42cc-8df5-b98b8aa1c167\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.171736239Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"datafile"} {"msg":"dataset is oxp_3e944a96-ab1b-47aa-a595-7e8c9f9e5446/crucible/regions/0b612c0b-3943-42cc-8df5-b98b8aa1c167","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.178552759Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"datafile"} {"msg":"snapshot is oxp_3e944a96-ab1b-47aa-a595-7e8c9f9e5446/crucible/regions/0b612c0b-3943-42cc-8df5-b98b8aa1c167@f525cb9e-65ff-4e02-943f-e77f4719895c","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.185886415Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_3e944a96-ab1b-47aa-a595-7e8c9f9e5446/crucible/regions/0b612c0b-3943-42cc-8df5-b98b8aa1c167\", \"f525cb9e-65ff-4e02-943f-e77f4719895c\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.185914465Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 0b612c0b-3943-42cc-8df5-b98b8aa1c167-f525cb9e-65ff-4e02-943f-e77f4719895c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.192986118Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"datafile"} {"msg":"applying SMF actions for region 0b612c0b-3943-42cc-8df5-b98b8aa1c167 running snapshot f525cb9e-65ff-4e02-943f-e77f4719895c (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.19313401Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.193143871Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"uri":"/crucible/0/regions/0b612c0b-3943-42cc-8df5-b98b8aa1c167/snapshots/f525cb9e-65ff-4e02-943f-e77f4719895c/run","method":"POST","req_id":"ec1c7d98-f03b-4ea6-9321-4b71b94dd620","remote_addr":"[fd00:1122:3344:101::a]:48256","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":21460,"response_code":200} {"msg":"creating missing snapshot instance snapshot-0b612c0b-3943-42cc-8df5-b98b8aa1c167-f525cb9e-65ff-4e02-943f-e77f4719895c","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.19818317Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-0b612c0b-3943-42cc-8df5-b98b8aa1c167-f525cb9e-65ff-4e02-943f-e77f4719895c","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.207111825Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-0b612c0b-3943-42cc-8df5-b98b8aa1c167-f525cb9e-65ff-4e02-943f-e77f4719895c","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.211953522Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0b612c0b-3943-42cc-8df5-b98b8aa1c167/.zfs/snapshot/f525cb9e-65ff-4e02-943f-e77f4719895c","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.212275456Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.212320426Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.212349326Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.212390087Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.212433987Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.21441003Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.217213563Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"worker"} {"msg":"region 0b612c0b-3943-42cc-8df5-b98b8aa1c167 running snapshot f525cb9e-65ff-4e02-943f-e77f4719895c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.217229414Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"datafile"} {"msg":"path is \"/data/regions/0b612c0b-3943-42cc-8df5-b98b8aa1c167\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.336392365Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"datafile"} {"msg":"dataset is oxp_3e944a96-ab1b-47aa-a595-7e8c9f9e5446/crucible/regions/0b612c0b-3943-42cc-8df5-b98b8aa1c167","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.34280201Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"datafile"} {"msg":"snapshot is oxp_3e944a96-ab1b-47aa-a595-7e8c9f9e5446/crucible/regions/0b612c0b-3943-42cc-8df5-b98b8aa1c167@f525cb9e-65ff-4e02-943f-e77f4719895c","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.349650721Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_3e944a96-ab1b-47aa-a595-7e8c9f9e5446/crucible/regions/0b612c0b-3943-42cc-8df5-b98b8aa1c167\", \"f525cb9e-65ff-4e02-943f-e77f4719895c\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.349680671Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:53:21.355239297Z","hostname":"oxz_crucible_a593bacd-d9ae-4be9-b42e-b096290383e9","pid":21789,"uri":"/crucible/0/regions/0b612c0b-3943-42cc-8df5-b98b8aa1c167/snapshots/f525cb9e-65ff-4e02-943f-e77f4719895c/run","method":"POST","req_id":"e0eebd03-a7cf-434d-8037-3f28983ec506","remote_addr":"[fd00:1122:3344:101::a]:48256","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":18888,"response_code":200}