| 1 | 2026-05-27T01:16:14.347Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::13]:32345
remote_addr = [fd00:1122:3344:101::c]:35526
|
| 2 | 2026-05-27T01:16:14.347Z | INFO | crucible-agent (datafile): region 81f16c95-92ad-4660-b56e-914e08ea7e32 state: Requested
|
| 3 | 2026-05-27T01:16:14.349Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 4 | 2026-05-27T01:16:14.349Z | INFO | crucible-agent (dropshot): request completed
latency_us = 901
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::c]:35526
req_id = 26b2f64d-35bd-4114-b571-515b60d5f3cb
response_code = 200
uri = /crucible/0/regions
|
| 5 | 2026-05-27T01:16:14.359Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_0e37c0f3-4aa3-423e-bdb4-a6a4ff0607f6/crucible/regions/81f16c95-92ad-4660-b56e-914e08ea7e32
|
| 6 | 2026-05-27T01:16:14.359Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_0e37c0f3-4aa3-423e-bdb4-a6a4ff0607f6/crucible/regions/81f16c95-92ad-4660-b56e-914e08ea7e32
|
| 7 | 2026-05-27T01:16:14.418Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("81f16c95-92ad-4660-b56e-914e08ea7e32"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/81f16c95-92ad-4660-b56e-914e08ea7e32"
region = 81f16c95-92ad-4660-b56e-914e08ea7e32
|
| 8 | 2026-05-27T01:16:14.418Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "81f16c95-92ad-4660-b56e-914e08ea7e32" "--data" "/data/regions/81f16c95-92ad-4660-b56e-914e08ea7e32" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = 81f16c95-92ad-4660-b56e-914e08ea7e32
|
| 9 | 2026-05-27T01:16:14.490Z | INFO | crucible-agent (worker): region files created ok
region = 81f16c95-92ad-4660-b56e-914e08ea7e32
|
| 10 | 2026-05-27T01:16:14.490Z | INFO | crucible-agent (datafile): region 81f16c95-92ad-4660-b56e-914e08ea7e32 state: Requested -> Created
|
| 11 | 2026-05-27T01:16:14.490Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 12 | 2026-05-27T01:16:14.491Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-81f16c95-92ad-4660-b56e-914e08ea7e32
|
| 13 | 2026-05-27T01:16:14.495Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-81f16c95-92ad-4660-b56e-914e08ea7e32
|
| 14 | 2026-05-27T01:16:14.510Z | INFO | crucible-agent (worker): creating config property group
|
| 15 | 2026-05-27T01:16:14.511Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-81f16c95-92ad-4660-b56e-914e08ea7e32
|
| 16 | 2026-05-27T01:16:14.512Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/81f16c95-92ad-4660-b56e-914e08ea7e32
|
| 17 | 2026-05-27T01:16:14.512Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19001
|
| 18 | 2026-05-27T01:16:14.512Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13
|
| 19 | 2026-05-27T01:16:14.512Z | INFO | crucible-agent (worker): commit
|
| 20 | 2026-05-27T01:16:14.514Z | INFO | crucible-agent (worker): ok!
|
| 21 | 2026-05-27T01:16:14.526Z | INFO | crucible-agent (worker): SMF ok!
|
| 22 | 2026-05-27T01:16:14.534Z | INFO | crucible-agent (dropshot): request completed
latency_us = 95
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::c]:35526
req_id = 7769fc27-b152-42a7-8140-22f9b2d6a1d3
response_code = 200
uri = /crucible/0/regions
|
| 23 | 2026-05-27T01:16:16.119Z | INFO | crucible-agent (dropshot): request completed
latency_us = 50
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::c]:35526
req_id = 2d3b7e4f-db8d-4438-b481-8c7fa259b581
response_code = 200
uri = /crucible/0/regions/97673d20-bedb-43ff-afa8-4366368e85c4
|
| 24 | 2026-05-27T01:16:16.123Z | INFO | crucible-agent (datafile): path is "/data/regions/97673d20-bedb-43ff-afa8-4366368e85c4"
|
| 25 | 2026-05-27T01:16:16.129Z | INFO | crucible-agent (datafile): dataset is oxp_0e37c0f3-4aa3-423e-bdb4-a6a4ff0607f6/crucible/regions/97673d20-bedb-43ff-afa8-4366368e85c4
|
| 26 | 2026-05-27T01:16:16.134Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_0e37c0f3-4aa3-423e-bdb4-a6a4ff0607f6/crucible/regions/97673d20-bedb-43ff-afa8-4366368e85c4@a24c66e7-f156-4735-bdcf-7d6bce1e2887
|
| 27 | 2026-05-27T01:16:16.134Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_0e37c0f3-4aa3-423e-bdb4-a6a4ff0607f6/crucible/regions/97673d20-bedb-43ff-afa8-4366368e85c4", "a24c66e7-f156-4735-bdcf-7d6bce1e2887"]
|
| 28 | 2026-05-27T01:16:16.141Z | INFO | crucible-agent (dropshot): request completed
latency_us = 18694
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::c]:35526
req_id = ff5f60bb-d00a-4f28-8fc1-291c954558f4
response_code = 200
uri = /crucible/0/regions/97673d20-bedb-43ff-afa8-4366368e85c4/snapshots/a24c66e7-f156-4735-bdcf-7d6bce1e2887
|
| 29 | 2026-05-27T01:16:16.145Z | INFO | crucible-agent (datafile): path is "/data/regions/97673d20-bedb-43ff-afa8-4366368e85c4"
|
| 30 | 2026-05-27T01:16:16.152Z | INFO | crucible-agent (datafile): dataset is oxp_0e37c0f3-4aa3-423e-bdb4-a6a4ff0607f6/crucible/regions/97673d20-bedb-43ff-afa8-4366368e85c4
|
| 31 | 2026-05-27T01:16:16.157Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_0e37c0f3-4aa3-423e-bdb4-a6a4ff0607f6/crucible/regions/97673d20-bedb-43ff-afa8-4366368e85c4@a24c66e7-f156-4735-bdcf-7d6bce1e2887
|
| 32 | 2026-05-27T01:16:16.157Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_0e37c0f3-4aa3-423e-bdb4-a6a4ff0607f6/crucible/regions/97673d20-bedb-43ff-afa8-4366368e85c4", "a24c66e7-f156-4735-bdcf-7d6bce1e2887"]
|
| 33 | 2026-05-27T01:16:16.163Z | INFO | crucible-agent (datafile): requesting running snapshot 97673d20-bedb-43ff-afa8-4366368e85c4-a24c66e7-f156-4735-bdcf-7d6bce1e2887 state: Requested
|
| 34 | 2026-05-27T01:16:16.163Z | INFO | crucible-agent (worker): applying SMF actions for region 97673d20-bedb-43ff-afa8-4366368e85c4 running snapshot a24c66e7-f156-4735-bdcf-7d6bce1e2887 (state Requested)...
|
| 35 | 2026-05-27T01:16:16.163Z | INFO | crucible-agent (dropshot): request completed
latency_us = 17966
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::c]:35526
req_id = fbadfd59-db2e-4c71-a271-0f2050fdb528
response_code = 200
uri = /crucible/0/regions/97673d20-bedb-43ff-afa8-4366368e85c4/snapshots/a24c66e7-f156-4735-bdcf-7d6bce1e2887/run
|
| 36 | 2026-05-27T01:16:16.168Z | INFO | crucible-agent (worker): creating missing snapshot instance snapshot-97673d20-bedb-43ff-afa8-4366368e85c4-a24c66e7-f156-4735-bdcf-7d6bce1e2887
|
| 37 | 2026-05-27T01:16:16.177Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:snapshot-97673d20-bedb-43ff-afa8-4366368e85c4-a24c66e7-f156-4735-bdcf-7d6bce1e2887
|
| 38 | 2026-05-27T01:16:16.182Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:snapshot-97673d20-bedb-43ff-afa8-4366368e85c4-a24c66e7-f156-4735-bdcf-7d6bce1e2887
|
| 39 | 2026-05-27T01:16:16.182Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/97673d20-bedb-43ff-afa8-4366368e85c4/.zfs/snapshot/a24c66e7-f156-4735-bdcf-7d6bce1e2887
|
| 40 | 2026-05-27T01:16:16.182Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19002
|
| 41 | 2026-05-27T01:16:16.182Z | INFO | crucible-agent (worker): ensure mode SCF_TYPE_ASTRING ro
|
| 42 | 2026-05-27T01:16:16.182Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13
|
| 43 | 2026-05-27T01:16:16.182Z | INFO | crucible-agent (worker): commit
|
| 44 | 2026-05-27T01:16:16.184Z | INFO | crucible-agent (worker): ok!
|
| 45 | 2026-05-27T01:16:16.186Z | INFO | crucible-agent (worker): SMF ok!
|
| 46 | 2026-05-27T01:16:16.186Z | INFO | crucible-agent (datafile): region 97673d20-bedb-43ff-afa8-4366368e85c4 running snapshot a24c66e7-f156-4735-bdcf-7d6bce1e2887 state: Requested -> Created
|
| 47 | 2026-05-27T01:16:16.436Z | INFO | crucible-agent (datafile): path is "/data/regions/97673d20-bedb-43ff-afa8-4366368e85c4"
|
| 48 | 2026-05-27T01:16:16.443Z | INFO | crucible-agent (datafile): dataset is oxp_0e37c0f3-4aa3-423e-bdb4-a6a4ff0607f6/crucible/regions/97673d20-bedb-43ff-afa8-4366368e85c4
|
| 49 | 2026-05-27T01:16:16.448Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_0e37c0f3-4aa3-423e-bdb4-a6a4ff0607f6/crucible/regions/97673d20-bedb-43ff-afa8-4366368e85c4@a24c66e7-f156-4735-bdcf-7d6bce1e2887
|
| 50 | 2026-05-27T01:16:16.448Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_0e37c0f3-4aa3-423e-bdb4-a6a4ff0607f6/crucible/regions/97673d20-bedb-43ff-afa8-4366368e85c4", "a24c66e7-f156-4735-bdcf-7d6bce1e2887"]
|
| 51 | 2026-05-27T01:16:16.452Z | INFO | crucible-agent (dropshot): request completed
latency_us = 16299
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::c]:35526
req_id = 19dea20e-44e1-4adc-838d-366f361d6232
response_code = 200
uri = /crucible/0/regions/97673d20-bedb-43ff-afa8-4366368e85c4/snapshots/a24c66e7-f156-4735-bdcf-7d6bce1e2887/run
|
| 52 | 2026-05-27T01:16:20.767Z | INFO | crucible-agent (datafile): region 3c4e5c27-804b-419f-976c-979577d6d029 state: Requested
|
| 53 | 2026-05-27T01:16:20.767Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 54 | 2026-05-27T01:16:20.767Z | INFO | crucible-agent (dropshot): request completed
latency_us = 219
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::c]:35526
req_id = bd6e70d0-b201-49ae-b24b-8bb906325afd
response_code = 200
uri = /crucible/0/regions
|
| 55 | 2026-05-27T01:16:20.774Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_0e37c0f3-4aa3-423e-bdb4-a6a4ff0607f6/crucible/regions/3c4e5c27-804b-419f-976c-979577d6d029
|
| 56 | 2026-05-27T01:16:20.774Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_0e37c0f3-4aa3-423e-bdb4-a6a4ff0607f6/crucible/regions/3c4e5c27-804b-419f-976c-979577d6d029
|
| 57 | 2026-05-27T01:16:20.816Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("3c4e5c27-804b-419f-976c-979577d6d029"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/3c4e5c27-804b-419f-976c-979577d6d029"
region = 3c4e5c27-804b-419f-976c-979577d6d029
|
| 58 | 2026-05-27T01:16:20.816Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "3c4e5c27-804b-419f-976c-979577d6d029" "--data" "/data/regions/3c4e5c27-804b-419f-976c-979577d6d029" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = 3c4e5c27-804b-419f-976c-979577d6d029
|
| 59 | 2026-05-27T01:16:20.868Z | INFO | crucible-agent (worker): region files created ok
region = 3c4e5c27-804b-419f-976c-979577d6d029
|
| 60 | 2026-05-27T01:16:20.869Z | INFO | crucible-agent (datafile): region 3c4e5c27-804b-419f-976c-979577d6d029 state: Requested -> Created
|
| 61 | 2026-05-27T01:16:20.869Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 62 | 2026-05-27T01:16:20.869Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-3c4e5c27-804b-419f-976c-979577d6d029
|
| 63 | 2026-05-27T01:16:20.870Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-3c4e5c27-804b-419f-976c-979577d6d029
|
| 64 | 2026-05-27T01:16:20.874Z | INFO | crucible-agent (worker): creating config property group
|
| 65 | 2026-05-27T01:16:20.875Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-3c4e5c27-804b-419f-976c-979577d6d029
|
| 66 | 2026-05-27T01:16:20.876Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/3c4e5c27-804b-419f-976c-979577d6d029
|
| 67 | 2026-05-27T01:16:20.876Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19003
|
| 68 | 2026-05-27T01:16:20.876Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13
|
| 69 | 2026-05-27T01:16:20.876Z | INFO | crucible-agent (worker): commit
|
| 70 | 2026-05-27T01:16:20.878Z | INFO | crucible-agent (worker): ok!
|
| 71 | 2026-05-27T01:16:20.885Z | INFO | crucible-agent (worker): SMF ok!
|
| 72 | 2026-05-27T01:16:20.964Z | INFO | crucible-agent (dropshot): request completed
latency_us = 78
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::c]:35526
req_id = 6f64d55a-1b9e-4f2d-92b2-66341eb80a82
response_code = 200
uri = /crucible/0/regions
|
| 73 | 2026-05-27T01:16:46.099Z | INFO | crucible-agent (dropshot): request completed
latency_us = 89
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::c]:35526
req_id = f7835107-af88-46dc-bd41-e318dbe7c147
response_code = 200
uri = /crucible/0/regions/3c4e5c27-804b-419f-976c-979577d6d029
|
| 74 | 2026-05-27T01:16:46.104Z | INFO | crucible-agent (datafile): path is "/data/regions/3c4e5c27-804b-419f-976c-979577d6d029"
|
| 75 | 2026-05-27T01:16:46.114Z | INFO | crucible-agent (datafile): dataset is oxp_0e37c0f3-4aa3-423e-bdb4-a6a4ff0607f6/crucible/regions/3c4e5c27-804b-419f-976c-979577d6d029
|
| 76 | 2026-05-27T01:16:46.120Z | INFO | crucible-agent (datafile): region 3c4e5c27-804b-419f-976c-979577d6d029 state: Created -> Tombstoned
|
| 77 | 2026-05-27T01:16:46.120Z | INFO | crucible-agent (worker): applying SMF actions before removal...
|
| 78 | 2026-05-27T01:16:46.120Z | INFO | crucible-agent (dropshot): request completed
latency_us = 16028
local_addr = [fd00:1122:3344:101::13]:32345
method = DELETE
remote_addr = [fd00:1122:3344:101::c]:35526
req_id = c50f112a-dcfb-4c80-a818-be853e3d3908
response_code = 204
uri = /crucible/0/regions/3c4e5c27-804b-419f-976c-979577d6d029
|
| 79 | 2026-05-27T01:16:46.122Z | INFO | crucible-agent (worker): disabling downstairs instance: downstairs-3c4e5c27-804b-419f-976c-979577d6d029 (instance states: (Some(Online), None))
|
| 80 | 2026-05-27T01:16:46.124Z | INFO | crucible-agent (dropshot): request completed
latency_us = 34
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::c]:35526
req_id = e0c4bdf3-a558-4a2b-8802-b32a29afd3bf
response_code = 200
uri = /crucible/0/regions/3c4e5c27-804b-419f-976c-979577d6d029
|
| 81 | 2026-05-27T01:16:46.127Z | INFO | crucible-agent (worker): SMF ok!
|
| 82 | 2026-05-27T01:16:46.135Z | INFO | crucible-agent (worker): deleting zfs dataset "oxp_0e37c0f3-4aa3-423e-bdb4-a6a4ff0607f6/crucible/regions/3c4e5c27-804b-419f-976c-979577d6d029"
region = 3c4e5c27-804b-419f-976c-979577d6d029
|
| 83 | 2026-05-27T01:16:46.181Z | INFO | crucible-agent (datafile): region 3c4e5c27-804b-419f-976c-979577d6d029 state: Tombstoned -> Destroyed
|
| 84 | 2026-05-27T01:16:46.186Z | INFO | crucible-agent (dropshot): request completed
latency_us = 80
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::c]:35526
req_id = c229c873-1a41-4fe9-8728-8007b6a3a8f5
response_code = 200
uri = /crucible/0/regions/3c4e5c27-804b-419f-976c-979577d6d029
|