{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.206731682Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:53478"} {"msg":"region 1aff98d8-0d5b-4b07-b67a-387e29e3672f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.207053881Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.207599896Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.207609376Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"uri":"/crucible/0/regions","method":"POST","req_id":"b3426f70-bed0-4af0-80f3-0192b264241a","remote_addr":"[fd00:1122:3344:101::b]:53478","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":597,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_423d57ae-147d-4dfc-b0f3-47c12f517139/crucible/regions/1aff98d8-0d5b-4b07-b67a-387e29e3672f","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.215694189Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_423d57ae-147d-4dfc-b0f3-47c12f517139/crucible/regions/1aff98d8-0d5b-4b07-b67a-387e29e3672f","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.215711409Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"1aff98d8-0d5b-4b07-b67a-387e29e3672f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1aff98d8-0d5b-4b07-b67a-387e29e3672f\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.260239643Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"region":"1aff98d8-0d5b-4b07-b67a-387e29e3672f","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1aff98d8-0d5b-4b07-b67a-387e29e3672f\" \"--data\" \"/data/regions/1aff98d8-0d5b-4b07-b67a-387e29e3672f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.260268033Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"region":"1aff98d8-0d5b-4b07-b67a-387e29e3672f","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.359652922Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"region":"1aff98d8-0d5b-4b07-b67a-387e29e3672f","component":"worker"} {"msg":"region 1aff98d8-0d5b-4b07-b67a-387e29e3672f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.359680872Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.359770365Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-1aff98d8-0d5b-4b07-b67a-387e29e3672f","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.379355571Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1aff98d8-0d5b-4b07-b67a-387e29e3672f","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.38502045Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.394009467Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1aff98d8-0d5b-4b07-b67a-387e29e3672f","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.397061487Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1aff98d8-0d5b-4b07-b67a-387e29e3672f","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.397341065Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.397392906Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.397430337Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.397466108Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.401534905Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.406230949Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:16.432910932Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"uri":"/crucible/0/regions","method":"POST","req_id":"812c5192-516d-4ae6-9520-b51c2fe778a9","remote_addr":"[fd00:1122:3344:101::b]:53478","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":60,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.58759791Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"uri":"/crucible/0/regions/01bade03-b48e-4f8b-9dc6-b091a9926ced","method":"GET","req_id":"8ef5010b-93cb-4943-9ed6-8a5f032d1337","remote_addr":"[fd00:1122:3344:101::b]:53478","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":75,"response_code":200} {"msg":"path is \"/data/regions/01bade03-b48e-4f8b-9dc6-b091a9926ced\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.591097413Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"datafile"} {"msg":"dataset is oxp_423d57ae-147d-4dfc-b0f3-47c12f517139/crucible/regions/01bade03-b48e-4f8b-9dc6-b091a9926ced","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.598127388Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"datafile"} {"msg":"snapshot is oxp_423d57ae-147d-4dfc-b0f3-47c12f517139/crucible/regions/01bade03-b48e-4f8b-9dc6-b091a9926ced@e45c57bd-e6bc-42cf-9e9d-3dfaba1ec498","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.604425294Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_423d57ae-147d-4dfc-b0f3-47c12f517139/crucible/regions/01bade03-b48e-4f8b-9dc6-b091a9926ced\", \"e45c57bd-e6bc-42cf-9e9d-3dfaba1ec498\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.604449084Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.612896817Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"uri":"/crucible/0/regions/01bade03-b48e-4f8b-9dc6-b091a9926ced/snapshots/e45c57bd-e6bc-42cf-9e9d-3dfaba1ec498","method":"GET","req_id":"28db57db-adb4-42c9-b14b-7beb7f8e9e0a","remote_addr":"[fd00:1122:3344:101::b]:53478","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":21825,"response_code":200} {"msg":"path is \"/data/regions/01bade03-b48e-4f8b-9dc6-b091a9926ced\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.616204784Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"datafile"} {"msg":"dataset is oxp_423d57ae-147d-4dfc-b0f3-47c12f517139/crucible/regions/01bade03-b48e-4f8b-9dc6-b091a9926ced","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.621243287Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"datafile"} {"msg":"snapshot is oxp_423d57ae-147d-4dfc-b0f3-47c12f517139/crucible/regions/01bade03-b48e-4f8b-9dc6-b091a9926ced@e45c57bd-e6bc-42cf-9e9d-3dfaba1ec498","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.628495918Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_423d57ae-147d-4dfc-b0f3-47c12f517139/crucible/regions/01bade03-b48e-4f8b-9dc6-b091a9926ced\", \"e45c57bd-e6bc-42cf-9e9d-3dfaba1ec498\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.628517688Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 01bade03-b48e-4f8b-9dc6-b091a9926ced-e45c57bd-e6bc-42cf-9e9d-3dfaba1ec498 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.635357509Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"datafile"} {"msg":"applying SMF actions for region 01bade03-b48e-4f8b-9dc6-b091a9926ced running snapshot e45c57bd-e6bc-42cf-9e9d-3dfaba1ec498 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.635480162Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.635490172Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"uri":"/crucible/0/regions/01bade03-b48e-4f8b-9dc6-b091a9926ced/snapshots/e45c57bd-e6bc-42cf-9e9d-3dfaba1ec498/run","method":"POST","req_id":"eb1584bd-15c3-447e-8abf-e0f3ec041616","remote_addr":"[fd00:1122:3344:101::b]:53478","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":19293,"response_code":200} {"msg":"creating missing snapshot instance snapshot-01bade03-b48e-4f8b-9dc6-b091a9926ced-e45c57bd-e6bc-42cf-9e9d-3dfaba1ec498","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.640019532Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-01bade03-b48e-4f8b-9dc6-b091a9926ced-e45c57bd-e6bc-42cf-9e9d-3dfaba1ec498","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.656557258Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-01bade03-b48e-4f8b-9dc6-b091a9926ced-e45c57bd-e6bc-42cf-9e9d-3dfaba1ec498","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.675595579Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/01bade03-b48e-4f8b-9dc6-b091a9926ced/.zfs/snapshot/e45c57bd-e6bc-42cf-9e9d-3dfaba1ec498","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.675795494Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.675838626Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.675875027Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.675909408Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.675953498Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.690231935Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.70255389Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"worker"} {"msg":"region 01bade03-b48e-4f8b-9dc6-b091a9926ced running snapshot e45c57bd-e6bc-42cf-9e9d-3dfaba1ec498 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.70256668Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"datafile"} {"msg":"path is \"/data/regions/01bade03-b48e-4f8b-9dc6-b091a9926ced\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.933038724Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"datafile"} {"msg":"dataset is oxp_423d57ae-147d-4dfc-b0f3-47c12f517139/crucible/regions/01bade03-b48e-4f8b-9dc6-b091a9926ced","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.939644467Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"datafile"} {"msg":"snapshot is oxp_423d57ae-147d-4dfc-b0f3-47c12f517139/crucible/regions/01bade03-b48e-4f8b-9dc6-b091a9926ced@e45c57bd-e6bc-42cf-9e9d-3dfaba1ec498","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.945845691Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_423d57ae-147d-4dfc-b0f3-47c12f517139/crucible/regions/01bade03-b48e-4f8b-9dc6-b091a9926ced\", \"e45c57bd-e6bc-42cf-9e9d-3dfaba1ec498\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.945877112Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-18T21:33:17.951655274Z","hostname":"oxz_crucible_92006cd6-a7f5-4186-892f-ba9ff055c8f2","pid":23857,"uri":"/crucible/0/regions/01bade03-b48e-4f8b-9dc6-b091a9926ced/snapshots/e45c57bd-e6bc-42cf-9e9d-3dfaba1ec498/run","method":"POST","req_id":"af809bc5-bc6d-40e0-a9cb-bda7d0d5a2ef","remote_addr":"[fd00:1122:3344:101::b]:53478","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":18661,"response_code":200}