[ Jan 16 03:31:45 Enabled. ] [ Jan 16 03:31:46 Rereading configuration. ] [ Jan 16 03:31:48 Rereading configuration. ] [ Jan 16 03:31:55 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:31:55.831967764Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992} {"msg":"dataset: \"oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:31:55.832862228Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992} {"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:31:55.832876308Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:31:55.832884648Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:31:55.845065771Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:31:56.110699094Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.206400088Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:35902"} {"msg":"region b56822f3-174f-4aea-abb1-776b488bd19d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.23656215Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.239534074Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.239589595Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions","method":"POST","req_id":"11b40d99-e91c-4ef4-9fd5-1fb9e2a4a471","remote_addr":"[fd00:1122:3344:101::c]:35902","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":16621,"response_code":200} {"msg":"zfs set reservation of 1342177280 for oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/b56822f3-174f-4aea-abb1-776b488bd19d","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.247731037Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/b56822f3-174f-4aea-abb1-776b488bd19d","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.247745567Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"b56822f3-174f-4aea-abb1-776b488bd19d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b56822f3-174f-4aea-abb1-776b488bd19d\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.3034304Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"region":"b56822f3-174f-4aea-abb1-776b488bd19d","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b56822f3-174f-4aea-abb1-776b488bd19d\" \"--data\" \"/data/regions/b56822f3-174f-4aea-abb1-776b488bd19d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.305637323Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"region":"b56822f3-174f-4aea-abb1-776b488bd19d","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.456223796Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions","method":"POST","req_id":"5abe18ec-c044-438d-85b8-fb99b68adba1","remote_addr":"[fd00:1122:3344:101::c]:35902","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":61,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.65104686Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"region":"b56822f3-174f-4aea-abb1-776b488bd19d","component":"worker"} {"msg":"region b56822f3-174f-4aea-abb1-776b488bd19d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.651120501Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.651308484Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-b56822f3-174f-4aea-abb1-776b488bd19d","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.6516957Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b56822f3-174f-4aea-abb1-776b488bd19d","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.664141316Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.676261998Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b56822f3-174f-4aea-abb1-776b488bd19d","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.679289933Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b56822f3-174f-4aea-abb1-776b488bd19d","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.679983043Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.680050004Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.680091265Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.680124855Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.684032204Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:23.688157666Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:24.152612024Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions","method":"POST","req_id":"1d568893-6532-49ed-9c74-42f7582fd026","remote_addr":"[fd00:1122:3344:101::c]:35902","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":55,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:24.547688824Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions/b56822f3-174f-4aea-abb1-776b488bd19d","method":"GET","req_id":"5da43b0b-99f4-41f0-bbe1-d4129801fa7b","remote_addr":"[fd00:1122:3344:101::c]:35902","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":83,"response_code":200} {"msg":"path is \"/data/regions/b56822f3-174f-4aea-abb1-776b488bd19d\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:24.552464496Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"dataset is oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/b56822f3-174f-4aea-abb1-776b488bd19d","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:24.560286873Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"region b56822f3-174f-4aea-abb1-776b488bd19d state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:24.567872866Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:24.568080409Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions/b56822f3-174f-4aea-abb1-776b488bd19d","method":"DELETE","req_id":"50d6a74d-4e2e-4a37-9974-e14e361585ab","remote_addr":"[fd00:1122:3344:101::c]:35902","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":15624,"response_code":204} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:24.56814135Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-b56822f3-174f-4aea-abb1-776b488bd19d (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:24.569054774Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:24.573416389Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions/b56822f3-174f-4aea-abb1-776b488bd19d","method":"GET","req_id":"bbd3c9be-6ff5-40b1-b540-4f52f97e412c","remote_addr":"[fd00:1122:3344:101::c]:35902","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":34,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:24.573591061Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/b56822f3-174f-4aea-abb1-776b488bd19d\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:24.58282291Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"region":"b56822f3-174f-4aea-abb1-776b488bd19d","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:24.643826172Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions/b56822f3-174f-4aea-abb1-776b488bd19d","method":"GET","req_id":"a89682d0-cc43-433b-ae00-cdfc44da3411","remote_addr":"[fd00:1122:3344:101::c]:35902","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":51,"response_code":200} {"msg":"region b56822f3-174f-4aea-abb1-776b488bd19d state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:24.662574893Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:24.746202934Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions/b56822f3-174f-4aea-abb1-776b488bd19d","method":"GET","req_id":"859044d1-c9fa-4a06-95dc-620a3dfc79db","remote_addr":"[fd00:1122:3344:101::c]:35902","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":33,"response_code":200} {"msg":"region 42340be5-b7bc-4ebf-8a7b-46c9525e1a4b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.21380612Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.213991203Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.214033544Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions","method":"POST","req_id":"7470728b-17ca-43e0-83bc-6437f9ee1a94","remote_addr":"[fd00:1122:3344:101::c]:35902","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":237,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.219897381Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.219993563Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"42340be5-b7bc-4ebf-8a7b-46c9525e1a4b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.281840188Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"region":"42340be5-b7bc-4ebf-8a7b-46c9525e1a4b","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"42340be5-b7bc-4ebf-8a7b-46c9525e1a4b\" \"--data\" \"/data/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.281910809Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"region":"42340be5-b7bc-4ebf-8a7b-46c9525e1a4b","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.361417569Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"region":"42340be5-b7bc-4ebf-8a7b-46c9525e1a4b","component":"worker"} {"msg":"region 42340be5-b7bc-4ebf-8a7b-46c9525e1a4b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.361446699Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.361697003Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-42340be5-b7bc-4ebf-8a7b-46c9525e1a4b","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.363035653Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-42340be5-b7bc-4ebf-8a7b-46c9525e1a4b","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.366301122Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.3735346Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-42340be5-b7bc-4ebf-8a7b-46c9525e1a4b","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.376595606Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.376805049Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.376842319Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.37688085Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.37691342Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.380864459Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.38559278Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:32:27.440031015Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions","method":"POST","req_id":"cb9e536d-8158-427c-89e1-6d20b10f7674","remote_addr":"[fd00:1122:3344:101::c]:35902","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":56,"response_code":200} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:05.874007593Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:39709"} {"msg":"region dd798c18-4b00-41cb-934d-5b675889e48c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:05.875002578Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:05.875023219Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions","method":"POST","req_id":"20a18c31-986a-468f-be7b-a4e74e73c903","remote_addr":"[fd00:1122:3344:101::c]:39709","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":839,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:05.875041409Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/dd798c18-4b00-41cb-934d-5b675889e48c","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:05.883343103Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/dd798c18-4b00-41cb-934d-5b675889e48c","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:05.883372104Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"dd798c18-4b00-41cb-934d-5b675889e48c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/dd798c18-4b00-41cb-934d-5b675889e48c\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:05.926474148Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"region":"dd798c18-4b00-41cb-934d-5b675889e48c","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"dd798c18-4b00-41cb-934d-5b675889e48c\" \"--data\" \"/data/regions/dd798c18-4b00-41cb-934d-5b675889e48c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:05.926510209Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"region":"dd798c18-4b00-41cb-934d-5b675889e48c","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:06.029422478Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"region":"dd798c18-4b00-41cb-934d-5b675889e48c","component":"worker"} {"msg":"region dd798c18-4b00-41cb-934d-5b675889e48c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:06.029471389Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:06.029603791Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-dd798c18-4b00-41cb-934d-5b675889e48c","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:06.047023482Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-dd798c18-4b00-41cb-934d-5b675889e48c","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:06.06430571Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:06.076380541Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-dd798c18-4b00-41cb-934d-5b675889e48c","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:06.079314405Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/dd798c18-4b00-41cb-934d-5b675889e48c","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:06.079513978Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:06.079554879Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:06.079607019Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:06.07964567Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:06.083052391Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:06.087572428Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:06.116358049Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions","method":"POST","req_id":"406a8280-f931-4feb-a3d8-e41ab9947d30","remote_addr":"[fd00:1122:3344:101::c]:39709","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":59,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.478181531Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b","method":"GET","req_id":"632fcbab-3ba6-4098-bab2-2f74128aacfc","remote_addr":"[fd00:1122:3344:101::c]:39709","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":51,"response_code":200} {"msg":"path is \"/data/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.481729234Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"dataset is oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.487837406Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"snapshot is oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b@0bc422a3-9d51-48e9-a8f9-22b4a62ea95f","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.494001428Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b\", \"0bc422a3-9d51-48e9-a8f9-22b4a62ea95f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.494029308Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.501744254Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b/snapshots/0bc422a3-9d51-48e9-a8f9-22b4a62ea95f","method":"GET","req_id":"49939c81-c083-445f-84dc-6a5874283dfb","remote_addr":"[fd00:1122:3344:101::c]:39709","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":20038,"response_code":200} {"msg":"path is \"/data/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.505199255Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"dataset is oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.511075073Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"snapshot is oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b@0bc422a3-9d51-48e9-a8f9-22b4a62ea95f","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.517000812Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b\", \"0bc422a3-9d51-48e9-a8f9-22b4a62ea95f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.517018822Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 42340be5-b7bc-4ebf-8a7b-46c9525e1a4b-0bc422a3-9d51-48e9-a8f9-22b4a62ea95f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.523630171Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.523794744Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b/snapshots/0bc422a3-9d51-48e9-a8f9-22b4a62ea95f/run","method":"POST","req_id":"ce36c619-1a17-4ed7-9069-e8733155f769","remote_addr":"[fd00:1122:3344:101::c]:39709","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":18576,"response_code":200} {"msg":"applying SMF actions for region 42340be5-b7bc-4ebf-8a7b-46c9525e1a4b running snapshot 0bc422a3-9d51-48e9-a8f9-22b4a62ea95f (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.523820794Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-42340be5-b7bc-4ebf-8a7b-46c9525e1a4b-0bc422a3-9d51-48e9-a8f9-22b4a62ea95f","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.528806219Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-42340be5-b7bc-4ebf-8a7b-46c9525e1a4b-0bc422a3-9d51-48e9-a8f9-22b4a62ea95f","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.531752663Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-42340be5-b7bc-4ebf-8a7b-46c9525e1a4b-0bc422a3-9d51-48e9-a8f9-22b4a62ea95f","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.540907329Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b/.zfs/snapshot/0bc422a3-9d51-48e9-a8f9-22b4a62ea95f","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.541110643Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.541150513Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.541185994Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.541220594Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.541261665Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.54430856Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.548364151Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"region 42340be5-b7bc-4ebf-8a7b-46c9525e1a4b running snapshot 0bc422a3-9d51-48e9-a8f9-22b4a62ea95f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.548379271Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"path is \"/data/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.83503255Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"dataset is oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.840966549Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"snapshot is oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b@0bc422a3-9d51-48e9-a8f9-22b4a62ea95f","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.847567577Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b\", \"0bc422a3-9d51-48e9-a8f9-22b4a62ea95f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.847601168Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:07.852839366Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions/42340be5-b7bc-4ebf-8a7b-46c9525e1a4b/snapshots/0bc422a3-9d51-48e9-a8f9-22b4a62ea95f/run","method":"POST","req_id":"3b70617e-2b40-4d3f-a100-f34202f28c71","remote_addr":"[fd00:1122:3344:101::c]:39709","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":17852,"response_code":200} {"msg":"region 29efe938-be4f-4b93-9d33-0b978bef8a82 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.608085994Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.608249117Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.608259547Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions","method":"POST","req_id":"f1f433a6-0391-43dd-a5ca-aa53d0ab3ce2","remote_addr":"[fd00:1122:3344:101::c]:39709","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":214,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/29efe938-be4f-4b93-9d33-0b978bef8a82","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.614646883Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/29efe938-be4f-4b93-9d33-0b978bef8a82","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.614687813Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"29efe938-be4f-4b93-9d33-0b978bef8a82\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/29efe938-be4f-4b93-9d33-0b978bef8a82\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.659435633Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"region":"29efe938-be4f-4b93-9d33-0b978bef8a82","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"29efe938-be4f-4b93-9d33-0b978bef8a82\" \"--data\" \"/data/regions/29efe938-be4f-4b93-9d33-0b978bef8a82\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.659475043Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"region":"29efe938-be4f-4b93-9d33-0b978bef8a82","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.738471315Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"region":"29efe938-be4f-4b93-9d33-0b978bef8a82","component":"worker"} {"msg":"region 29efe938-be4f-4b93-9d33-0b978bef8a82 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.738514966Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.738620207Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-29efe938-be4f-4b93-9d33-0b978bef8a82","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.739537531Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-29efe938-be4f-4b93-9d33-0b978bef8a82","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.742256141Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.749132264Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-29efe938-be4f-4b93-9d33-0b978bef8a82","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.752304032Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/29efe938-be4f-4b93-9d33-0b978bef8a82","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.752568546Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.752618457Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.752658447Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.752694388Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.757080433Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.767173664Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:13.811380406Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions","method":"POST","req_id":"47c6296b-704d-47aa-a8b2-ef27a8bcd8df","remote_addr":"[fd00:1122:3344:101::c]:39709","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":51,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:42.100548135Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions/29efe938-be4f-4b93-9d33-0b978bef8a82","method":"GET","req_id":"bfbc3e42-bf89-4293-855d-4b1cdd528c9f","remote_addr":"[fd00:1122:3344:101::c]:39709","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":50,"response_code":200} {"msg":"path is \"/data/regions/29efe938-be4f-4b93-9d33-0b978bef8a82\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:42.103134543Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"dataset is oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/29efe938-be4f-4b93-9d33-0b978bef8a82","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:42.112596325Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"region 29efe938-be4f-4b93-9d33-0b978bef8a82 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:42.119156703Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:42.119402547Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:42.119413287Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions/29efe938-be4f-4b93-9d33-0b978bef8a82","method":"DELETE","req_id":"b1024b0e-e982-4577-8bbe-ddf9830019bf","remote_addr":"[fd00:1122:3344:101::c]:39709","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":16318,"response_code":204} {"msg":"disabling downstairs instance: downstairs-29efe938-be4f-4b93-9d33-0b978bef8a82 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:42.120679446Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:42.12296314Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions/29efe938-be4f-4b93-9d33-0b978bef8a82","method":"GET","req_id":"371c2de6-fc52-40c3-a7fa-6c8a4e053f3d","remote_addr":"[fd00:1122:3344:101::c]:39709","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":29,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:42.151828812Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_49c20d27-0b12-40d1-96ac-293312095c18/crucible/regions/29efe938-be4f-4b93-9d33-0b978bef8a82\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:42.157787801Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"region":"29efe938-be4f-4b93-9d33-0b978bef8a82","component":"worker"} {"msg":"region 29efe938-be4f-4b93-9d33-0b978bef8a82 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:42.245485133Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-16T03:44:42.276199503Z","hostname":"oxz_crucible_6bbe64d4-1c2a-46a1-a9c5-66eca93d97e0","pid":25992,"uri":"/crucible/0/regions/29efe938-be4f-4b93-9d33-0b978bef8a82","method":"GET","req_id":"c359e050-f152-4aa7-9391-9306715ad5c6","remote_addr":"[fd00:1122:3344:101::c]:39709","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":50,"response_code":200}