| 1 | 2026-05-21T10:22:18.440Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::13]:32345
remote_addr = [fd00:1122:3344:101::c]:45672
|
| 2 | 2026-05-21T10:22:18.440Z | INFO | crucible-agent (datafile): region e3d229a2-79c6-492b-8cfd-c2eff19ae887 state: Requested
|
| 3 | 2026-05-21T10:22:18.441Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 4 | 2026-05-21T10:22:18.441Z | INFO | crucible-agent (dropshot): request completed
latency_us = 1259
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::c]:45672
req_id = 3c291b4e-a4ff-42f6-be53-737b3aa275e5
response_code = 200
uri = /crucible/0/regions
|
| 5 | 2026-05-21T10:22:18.449Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_ea909ff9-a126-4b1b-9dcb-898a88caa1e7/crucible/regions/e3d229a2-79c6-492b-8cfd-c2eff19ae887
|
| 6 | 2026-05-21T10:22:18.449Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_ea909ff9-a126-4b1b-9dcb-898a88caa1e7/crucible/regions/e3d229a2-79c6-492b-8cfd-c2eff19ae887
|
| 7 | 2026-05-21T10:22:18.489Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("e3d229a2-79c6-492b-8cfd-c2eff19ae887"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/e3d229a2-79c6-492b-8cfd-c2eff19ae887"
region = e3d229a2-79c6-492b-8cfd-c2eff19ae887
|
| 8 | 2026-05-21T10:22:18.489Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "e3d229a2-79c6-492b-8cfd-c2eff19ae887" "--data" "/data/regions/e3d229a2-79c6-492b-8cfd-c2eff19ae887" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = e3d229a2-79c6-492b-8cfd-c2eff19ae887
|
| 9 | 2026-05-21T10:22:18.544Z | INFO | crucible-agent (worker): region files created ok
region = e3d229a2-79c6-492b-8cfd-c2eff19ae887
|
| 10 | 2026-05-21T10:22:18.544Z | INFO | crucible-agent (datafile): region e3d229a2-79c6-492b-8cfd-c2eff19ae887 state: Requested -> Created
|
| 11 | 2026-05-21T10:22:18.544Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 12 | 2026-05-21T10:22:18.544Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-e3d229a2-79c6-492b-8cfd-c2eff19ae887
|
| 13 | 2026-05-21T10:22:18.548Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-e3d229a2-79c6-492b-8cfd-c2eff19ae887
|
| 14 | 2026-05-21T10:22:18.559Z | INFO | crucible-agent (worker): creating config property group
|
| 15 | 2026-05-21T10:22:18.560Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-e3d229a2-79c6-492b-8cfd-c2eff19ae887
|
| 16 | 2026-05-21T10:22:18.561Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/e3d229a2-79c6-492b-8cfd-c2eff19ae887
|
| 17 | 2026-05-21T10:22:18.561Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19001
|
| 18 | 2026-05-21T10:22:18.561Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13
|
| 19 | 2026-05-21T10:22:18.561Z | INFO | crucible-agent (worker): commit
|
| 20 | 2026-05-21T10:22:18.563Z | INFO | crucible-agent (worker): ok!
|
| 21 | 2026-05-21T10:22:18.572Z | INFO | crucible-agent (worker): SMF ok!
|
| 22 | 2026-05-21T10:22:18.794Z | INFO | crucible-agent (dropshot): request completed
latency_us = 63
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::c]:45672
req_id = 135b3989-fde7-4989-adc1-e0a07432427e
response_code = 200
uri = /crucible/0/regions
|
| 23 | 2026-05-21T10:22:20.175Z | INFO | crucible-agent (dropshot): request completed
latency_us = 69
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::c]:45672
req_id = d8e47aca-1926-484f-aabf-4dd6eb4029d6
response_code = 200
uri = /crucible/0/regions/f7f52bcd-db56-4b04-9cb8-a10d05ae5881
|
| 24 | 2026-05-21T10:22:20.179Z | INFO | crucible-agent (datafile): path is "/data/regions/f7f52bcd-db56-4b04-9cb8-a10d05ae5881"
|
| 25 | 2026-05-21T10:22:20.184Z | INFO | crucible-agent (datafile): dataset is oxp_ea909ff9-a126-4b1b-9dcb-898a88caa1e7/crucible/regions/f7f52bcd-db56-4b04-9cb8-a10d05ae5881
|
| 26 | 2026-05-21T10:22:20.191Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_ea909ff9-a126-4b1b-9dcb-898a88caa1e7/crucible/regions/f7f52bcd-db56-4b04-9cb8-a10d05ae5881@0f6e2fea-5bcc-48bd-98c6-5c1e39b239a3
|
| 27 | 2026-05-21T10:22:20.191Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_ea909ff9-a126-4b1b-9dcb-898a88caa1e7/crucible/regions/f7f52bcd-db56-4b04-9cb8-a10d05ae5881", "0f6e2fea-5bcc-48bd-98c6-5c1e39b239a3"]
|
| 28 | 2026-05-21T10:22:20.199Z | INFO | crucible-agent (dropshot): request completed
latency_us = 20429
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::c]:45672
req_id = 03db64ef-c189-441e-9ae1-358199fc4108
response_code = 200
uri = /crucible/0/regions/f7f52bcd-db56-4b04-9cb8-a10d05ae5881/snapshots/0f6e2fea-5bcc-48bd-98c6-5c1e39b239a3
|
| 29 | 2026-05-21T10:22:20.203Z | INFO | crucible-agent (datafile): path is "/data/regions/f7f52bcd-db56-4b04-9cb8-a10d05ae5881"
|
| 30 | 2026-05-21T10:22:20.209Z | INFO | crucible-agent (datafile): dataset is oxp_ea909ff9-a126-4b1b-9dcb-898a88caa1e7/crucible/regions/f7f52bcd-db56-4b04-9cb8-a10d05ae5881
|
| 31 | 2026-05-21T10:22:20.214Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_ea909ff9-a126-4b1b-9dcb-898a88caa1e7/crucible/regions/f7f52bcd-db56-4b04-9cb8-a10d05ae5881@0f6e2fea-5bcc-48bd-98c6-5c1e39b239a3
|
| 32 | 2026-05-21T10:22:20.214Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_ea909ff9-a126-4b1b-9dcb-898a88caa1e7/crucible/regions/f7f52bcd-db56-4b04-9cb8-a10d05ae5881", "0f6e2fea-5bcc-48bd-98c6-5c1e39b239a3"]
|
| 33 | 2026-05-21T10:22:20.220Z | INFO | crucible-agent (datafile): requesting running snapshot f7f52bcd-db56-4b04-9cb8-a10d05ae5881-0f6e2fea-5bcc-48bd-98c6-5c1e39b239a3 state: Requested
|
| 34 | 2026-05-21T10:22:20.220Z | INFO | crucible-agent (dropshot): request completed
latency_us = 17444
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::c]:45672
req_id = bdeda09f-03eb-43d1-94c2-59922a75ac41
response_code = 200
uri = /crucible/0/regions/f7f52bcd-db56-4b04-9cb8-a10d05ae5881/snapshots/0f6e2fea-5bcc-48bd-98c6-5c1e39b239a3/run
|
| 35 | 2026-05-21T10:22:20.220Z | INFO | crucible-agent (worker): applying SMF actions for region f7f52bcd-db56-4b04-9cb8-a10d05ae5881 running snapshot 0f6e2fea-5bcc-48bd-98c6-5c1e39b239a3 (state Requested)...
|
| 36 | 2026-05-21T10:22:20.225Z | INFO | crucible-agent (worker): creating missing snapshot instance snapshot-f7f52bcd-db56-4b04-9cb8-a10d05ae5881-0f6e2fea-5bcc-48bd-98c6-5c1e39b239a3
|
| 37 | 2026-05-21T10:22:20.234Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:snapshot-f7f52bcd-db56-4b04-9cb8-a10d05ae5881-0f6e2fea-5bcc-48bd-98c6-5c1e39b239a3
|
| 38 | 2026-05-21T10:22:20.239Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:snapshot-f7f52bcd-db56-4b04-9cb8-a10d05ae5881-0f6e2fea-5bcc-48bd-98c6-5c1e39b239a3
|
| 39 | 2026-05-21T10:22:20.239Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/f7f52bcd-db56-4b04-9cb8-a10d05ae5881/.zfs/snapshot/0f6e2fea-5bcc-48bd-98c6-5c1e39b239a3
|
| 40 | 2026-05-21T10:22:20.239Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19002
|
| 41 | 2026-05-21T10:22:20.239Z | INFO | crucible-agent (worker): ensure mode SCF_TYPE_ASTRING ro
|
| 42 | 2026-05-21T10:22:20.239Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13
|
| 43 | 2026-05-21T10:22:20.239Z | INFO | crucible-agent (worker): commit
|
| 44 | 2026-05-21T10:22:20.241Z | INFO | crucible-agent (worker): ok!
|
| 45 | 2026-05-21T10:22:20.244Z | INFO | crucible-agent (worker): SMF ok!
|
| 46 | 2026-05-21T10:22:20.244Z | INFO | crucible-agent (datafile): region f7f52bcd-db56-4b04-9cb8-a10d05ae5881 running snapshot 0f6e2fea-5bcc-48bd-98c6-5c1e39b239a3 state: Requested -> Created
|
| 47 | 2026-05-21T10:22:20.374Z | INFO | crucible-agent (datafile): path is "/data/regions/f7f52bcd-db56-4b04-9cb8-a10d05ae5881"
|
| 48 | 2026-05-21T10:22:20.380Z | INFO | crucible-agent (datafile): dataset is oxp_ea909ff9-a126-4b1b-9dcb-898a88caa1e7/crucible/regions/f7f52bcd-db56-4b04-9cb8-a10d05ae5881
|
| 49 | 2026-05-21T10:22:20.386Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_ea909ff9-a126-4b1b-9dcb-898a88caa1e7/crucible/regions/f7f52bcd-db56-4b04-9cb8-a10d05ae5881@0f6e2fea-5bcc-48bd-98c6-5c1e39b239a3
|
| 50 | 2026-05-21T10:22:20.386Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_ea909ff9-a126-4b1b-9dcb-898a88caa1e7/crucible/regions/f7f52bcd-db56-4b04-9cb8-a10d05ae5881", "0f6e2fea-5bcc-48bd-98c6-5c1e39b239a3"]
|
| 51 | 2026-05-21T10:22:20.391Z | INFO | crucible-agent (dropshot): request completed
latency_us = 16554
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::c]:45672
req_id = b4674d88-f395-469f-a9b9-19397907d5b3
response_code = 200
uri = /crucible/0/regions/f7f52bcd-db56-4b04-9cb8-a10d05ae5881/snapshots/0f6e2fea-5bcc-48bd-98c6-5c1e39b239a3/run
|
| 52 | 2026-05-21T10:22:25.245Z | INFO | crucible-agent (datafile): region 1db103fd-5be9-4a3c-af45-2bef8a59bd0d state: Requested
|
| 53 | 2026-05-21T10:22:25.245Z | INFO | crucible-agent (dropshot): request completed
latency_us = 210
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::c]:45672
req_id = 92d65072-7aa1-42c9-a0ef-ebce5d9faf0c
response_code = 200
uri = /crucible/0/regions
|
| 54 | 2026-05-21T10:22:25.246Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 55 | 2026-05-21T10:22:25.251Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_ea909ff9-a126-4b1b-9dcb-898a88caa1e7/crucible/regions/1db103fd-5be9-4a3c-af45-2bef8a59bd0d
|
| 56 | 2026-05-21T10:22:25.252Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_ea909ff9-a126-4b1b-9dcb-898a88caa1e7/crucible/regions/1db103fd-5be9-4a3c-af45-2bef8a59bd0d
|
| 57 | 2026-05-21T10:22:25.289Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("1db103fd-5be9-4a3c-af45-2bef8a59bd0d"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/1db103fd-5be9-4a3c-af45-2bef8a59bd0d"
region = 1db103fd-5be9-4a3c-af45-2bef8a59bd0d
|
| 58 | 2026-05-21T10:22:25.289Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "1db103fd-5be9-4a3c-af45-2bef8a59bd0d" "--data" "/data/regions/1db103fd-5be9-4a3c-af45-2bef8a59bd0d" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = 1db103fd-5be9-4a3c-af45-2bef8a59bd0d
|
| 59 | 2026-05-21T10:22:25.340Z | INFO | crucible-agent (worker): region files created ok
region = 1db103fd-5be9-4a3c-af45-2bef8a59bd0d
|
| 60 | 2026-05-21T10:22:25.340Z | INFO | crucible-agent (datafile): region 1db103fd-5be9-4a3c-af45-2bef8a59bd0d state: Requested -> Created
|
| 61 | 2026-05-21T10:22:25.340Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 62 | 2026-05-21T10:22:25.340Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-1db103fd-5be9-4a3c-af45-2bef8a59bd0d
|
| 63 | 2026-05-21T10:22:25.341Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-1db103fd-5be9-4a3c-af45-2bef8a59bd0d
|
| 64 | 2026-05-21T10:22:25.345Z | INFO | crucible-agent (worker): creating config property group
|
| 65 | 2026-05-21T10:22:25.346Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-1db103fd-5be9-4a3c-af45-2bef8a59bd0d
|
| 66 | 2026-05-21T10:22:25.346Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/1db103fd-5be9-4a3c-af45-2bef8a59bd0d
|
| 67 | 2026-05-21T10:22:25.346Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19003
|
| 68 | 2026-05-21T10:22:25.346Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13
|
| 69 | 2026-05-21T10:22:25.346Z | INFO | crucible-agent (worker): commit
|
| 70 | 2026-05-21T10:22:25.348Z | INFO | crucible-agent (worker): ok!
|
| 71 | 2026-05-21T10:22:25.356Z | INFO | crucible-agent (worker): SMF ok!
|
| 72 | 2026-05-21T10:22:25.429Z | INFO | crucible-agent (dropshot): request completed
latency_us = 57
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::c]:45672
req_id = c96d5702-073e-4fe3-b01f-16b99dbc6536
response_code = 200
uri = /crucible/0/regions
|
| 73 | 2026-05-21T10:22:50.551Z | INFO | crucible-agent (dropshot): request completed
latency_us = 65
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::c]:45672
req_id = ac59c363-2f44-4fd2-b3b3-8be1b02a305a
response_code = 200
uri = /crucible/0/regions/1db103fd-5be9-4a3c-af45-2bef8a59bd0d
|
| 74 | 2026-05-21T10:22:50.556Z | INFO | crucible-agent (datafile): path is "/data/regions/1db103fd-5be9-4a3c-af45-2bef8a59bd0d"
|
| 75 | 2026-05-21T10:22:50.565Z | INFO | crucible-agent (datafile): dataset is oxp_ea909ff9-a126-4b1b-9dcb-898a88caa1e7/crucible/regions/1db103fd-5be9-4a3c-af45-2bef8a59bd0d
|
| 76 | 2026-05-21T10:22:50.572Z | INFO | crucible-agent (datafile): region 1db103fd-5be9-4a3c-af45-2bef8a59bd0d state: Created -> Tombstoned
|
| 77 | 2026-05-21T10:22:50.572Z | INFO | crucible-agent (worker): applying SMF actions before removal...
|
| 78 | 2026-05-21T10:22:50.572Z | INFO | crucible-agent (dropshot): request completed
latency_us = 16737
local_addr = [fd00:1122:3344:101::13]:32345
method = DELETE
remote_addr = [fd00:1122:3344:101::c]:45672
req_id = a77fa581-ea77-40dd-8720-f9219a472726
response_code = 204
uri = /crucible/0/regions/1db103fd-5be9-4a3c-af45-2bef8a59bd0d
|
| 79 | 2026-05-21T10:22:50.574Z | INFO | crucible-agent (worker): disabling downstairs instance: downstairs-1db103fd-5be9-4a3c-af45-2bef8a59bd0d (instance states: (Some(Online), None))
|
| 80 | 2026-05-21T10:22:50.578Z | INFO | crucible-agent (worker): SMF ok!
|
| 81 | 2026-05-21T10:22:50.580Z | INFO | crucible-agent (dropshot): request completed
latency_us = 75
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::c]:45672
req_id = 9eea25ee-acd5-4639-9c75-b22e17b8b1bb
response_code = 200
uri = /crucible/0/regions/1db103fd-5be9-4a3c-af45-2bef8a59bd0d
|
| 82 | 2026-05-21T10:22:50.587Z | INFO | crucible-agent (worker): deleting zfs dataset "oxp_ea909ff9-a126-4b1b-9dcb-898a88caa1e7/crucible/regions/1db103fd-5be9-4a3c-af45-2bef8a59bd0d"
region = 1db103fd-5be9-4a3c-af45-2bef8a59bd0d
|
| 83 | 2026-05-21T10:22:50.638Z | INFO | crucible-agent (datafile): region 1db103fd-5be9-4a3c-af45-2bef8a59bd0d state: Tombstoned -> Destroyed
|
| 84 | 2026-05-21T10:22:50.642Z | INFO | crucible-agent (dropshot): request completed
latency_us = 55
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::c]:45672
req_id = 1e00f73b-6b5f-4fad-97f2-ac3103a84484
response_code = 200
uri = /crucible/0/regions/1db103fd-5be9-4a3c-af45-2bef8a59bd0d
|