| 1 | 2026-05-14T16:08:34.146Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::1a]:32345
remote_addr = [fd00:1122:3344:101::a]:44389
|
| 2 | 2026-05-14T16:08:34.147Z | INFO | crucible-agent (datafile): region b8a772eb-57c8-41d4-8eef-d22c7cfc941a state: Requested
|
| 3 | 2026-05-14T16:08:34.147Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 4 | 2026-05-14T16:08:34.147Z | INFO | crucible-agent (dropshot): request completed
latency_us = 778
local_addr = [fd00:1122:3344:101::1a]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:44389
req_id = 02cc8b68-34c6-493f-bc0f-775a2607b673
response_code = 200
uri = /crucible/0/regions
|
| 5 | 2026-05-14T16:08:34.156Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_63d1fcbf-1b29-47c8-a6d4-05e3a30623cc/crucible/regions/b8a772eb-57c8-41d4-8eef-d22c7cfc941a
|
| 6 | 2026-05-14T16:08:34.156Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_63d1fcbf-1b29-47c8-a6d4-05e3a30623cc/crucible/regions/b8a772eb-57c8-41d4-8eef-d22c7cfc941a
|
| 7 | 2026-05-14T16:08:34.200Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("b8a772eb-57c8-41d4-8eef-d22c7cfc941a"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/b8a772eb-57c8-41d4-8eef-d22c7cfc941a"
region = b8a772eb-57c8-41d4-8eef-d22c7cfc941a
|
| 8 | 2026-05-14T16:08:34.200Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "b8a772eb-57c8-41d4-8eef-d22c7cfc941a" "--data" "/data/regions/b8a772eb-57c8-41d4-8eef-d22c7cfc941a" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = b8a772eb-57c8-41d4-8eef-d22c7cfc941a
|
| 9 | 2026-05-14T16:08:34.254Z | INFO | crucible-agent (worker): region files created ok
region = b8a772eb-57c8-41d4-8eef-d22c7cfc941a
|
| 10 | 2026-05-14T16:08:34.254Z | INFO | crucible-agent (datafile): region b8a772eb-57c8-41d4-8eef-d22c7cfc941a state: Requested -> Created
|
| 11 | 2026-05-14T16:08:34.254Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 12 | 2026-05-14T16:08:34.255Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-b8a772eb-57c8-41d4-8eef-d22c7cfc941a
|
| 13 | 2026-05-14T16:08:34.262Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-b8a772eb-57c8-41d4-8eef-d22c7cfc941a
|
| 14 | 2026-05-14T16:08:34.272Z | INFO | crucible-agent (worker): creating config property group
|
| 15 | 2026-05-14T16:08:34.274Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-b8a772eb-57c8-41d4-8eef-d22c7cfc941a
|
| 16 | 2026-05-14T16:08:34.274Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/b8a772eb-57c8-41d4-8eef-d22c7cfc941a
|
| 17 | 2026-05-14T16:08:34.274Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19001
|
| 18 | 2026-05-14T16:08:34.274Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a
|
| 19 | 2026-05-14T16:08:34.274Z | INFO | crucible-agent (worker): commit
|
| 20 | 2026-05-14T16:08:34.276Z | INFO | crucible-agent (worker): ok!
|
| 21 | 2026-05-14T16:08:34.287Z | INFO | crucible-agent (worker): SMF ok!
|
| 22 | 2026-05-14T16:08:34.513Z | INFO | crucible-agent (dropshot): request completed
latency_us = 78
local_addr = [fd00:1122:3344:101::1a]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:44389
req_id = 99424ee4-7ebe-4c10-a686-de403eb3c65b
response_code = 200
uri = /crucible/0/regions
|
| 23 | 2026-05-14T16:08:35.890Z | INFO | crucible-agent (dropshot): request completed
latency_us = 51
local_addr = [fd00:1122:3344:101::1a]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:44389
req_id = 37247b95-69a2-43b0-8438-861fdd468d83
response_code = 200
uri = /crucible/0/regions/f33a8e81-b1a1-4f0b-bf76-087f85098c9b
|
| 24 | 2026-05-14T16:08:35.894Z | INFO | crucible-agent (datafile): path is "/data/regions/f33a8e81-b1a1-4f0b-bf76-087f85098c9b"
|
| 25 | 2026-05-14T16:08:35.900Z | INFO | crucible-agent (datafile): dataset is oxp_63d1fcbf-1b29-47c8-a6d4-05e3a30623cc/crucible/regions/f33a8e81-b1a1-4f0b-bf76-087f85098c9b
|
| 26 | 2026-05-14T16:08:35.907Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_63d1fcbf-1b29-47c8-a6d4-05e3a30623cc/crucible/regions/f33a8e81-b1a1-4f0b-bf76-087f85098c9b@d9acf02d-46bc-424f-a213-37939f99a0a0
|
| 27 | 2026-05-14T16:08:35.907Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_63d1fcbf-1b29-47c8-a6d4-05e3a30623cc/crucible/regions/f33a8e81-b1a1-4f0b-bf76-087f85098c9b", "d9acf02d-46bc-424f-a213-37939f99a0a0"]
|
| 28 | 2026-05-14T16:08:35.915Z | INFO | crucible-agent (dropshot): request completed
latency_us = 20884
local_addr = [fd00:1122:3344:101::1a]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:44389
req_id = 5a4aa4fb-17d6-453f-9770-3e0b940c37d8
response_code = 200
uri = /crucible/0/regions/f33a8e81-b1a1-4f0b-bf76-087f85098c9b/snapshots/d9acf02d-46bc-424f-a213-37939f99a0a0
|
| 29 | 2026-05-14T16:08:35.919Z | INFO | crucible-agent (datafile): path is "/data/regions/f33a8e81-b1a1-4f0b-bf76-087f85098c9b"
|
| 30 | 2026-05-14T16:08:35.925Z | INFO | crucible-agent (datafile): dataset is oxp_63d1fcbf-1b29-47c8-a6d4-05e3a30623cc/crucible/regions/f33a8e81-b1a1-4f0b-bf76-087f85098c9b
|
| 31 | 2026-05-14T16:08:35.931Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_63d1fcbf-1b29-47c8-a6d4-05e3a30623cc/crucible/regions/f33a8e81-b1a1-4f0b-bf76-087f85098c9b@d9acf02d-46bc-424f-a213-37939f99a0a0
|
| 32 | 2026-05-14T16:08:35.931Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_63d1fcbf-1b29-47c8-a6d4-05e3a30623cc/crucible/regions/f33a8e81-b1a1-4f0b-bf76-087f85098c9b", "d9acf02d-46bc-424f-a213-37939f99a0a0"]
|
| 33 | 2026-05-14T16:08:35.936Z | INFO | crucible-agent (datafile): requesting running snapshot f33a8e81-b1a1-4f0b-bf76-087f85098c9b-d9acf02d-46bc-424f-a213-37939f99a0a0 state: Requested
|
| 34 | 2026-05-14T16:08:35.937Z | INFO | crucible-agent (dropshot): request completed
latency_us = 17808
local_addr = [fd00:1122:3344:101::1a]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:44389
req_id = 7e6b8e6b-65f0-4c81-b563-5d652a87c10c
response_code = 200
uri = /crucible/0/regions/f33a8e81-b1a1-4f0b-bf76-087f85098c9b/snapshots/d9acf02d-46bc-424f-a213-37939f99a0a0/run
|
| 35 | 2026-05-14T16:08:35.937Z | INFO | crucible-agent (worker): applying SMF actions for region f33a8e81-b1a1-4f0b-bf76-087f85098c9b running snapshot d9acf02d-46bc-424f-a213-37939f99a0a0 (state Requested)...
|
| 36 | 2026-05-14T16:08:35.942Z | INFO | crucible-agent (worker): creating missing snapshot instance snapshot-f33a8e81-b1a1-4f0b-bf76-087f85098c9b-d9acf02d-46bc-424f-a213-37939f99a0a0
|
| 37 | 2026-05-14T16:08:35.943Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:snapshot-f33a8e81-b1a1-4f0b-bf76-087f85098c9b-d9acf02d-46bc-424f-a213-37939f99a0a0
|
| 38 | 2026-05-14T16:08:35.948Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:snapshot-f33a8e81-b1a1-4f0b-bf76-087f85098c9b-d9acf02d-46bc-424f-a213-37939f99a0a0
|
| 39 | 2026-05-14T16:08:35.948Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/f33a8e81-b1a1-4f0b-bf76-087f85098c9b/.zfs/snapshot/d9acf02d-46bc-424f-a213-37939f99a0a0
|
| 40 | 2026-05-14T16:08:35.948Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19002
|
| 41 | 2026-05-14T16:08:35.948Z | INFO | crucible-agent (worker): ensure mode SCF_TYPE_ASTRING ro
|
| 42 | 2026-05-14T16:08:35.948Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a
|
| 43 | 2026-05-14T16:08:35.948Z | INFO | crucible-agent (worker): commit
|
| 44 | 2026-05-14T16:08:35.950Z | INFO | crucible-agent (worker): ok!
|
| 45 | 2026-05-14T16:08:35.953Z | INFO | crucible-agent (worker): SMF ok!
|
| 46 | 2026-05-14T16:08:35.953Z | INFO | crucible-agent (datafile): region f33a8e81-b1a1-4f0b-bf76-087f85098c9b running snapshot d9acf02d-46bc-424f-a213-37939f99a0a0 state: Requested -> Created
|
| 47 | 2026-05-14T16:08:36.257Z | INFO | crucible-agent (datafile): path is "/data/regions/f33a8e81-b1a1-4f0b-bf76-087f85098c9b"
|
| 48 | 2026-05-14T16:08:36.265Z | INFO | crucible-agent (datafile): dataset is oxp_63d1fcbf-1b29-47c8-a6d4-05e3a30623cc/crucible/regions/f33a8e81-b1a1-4f0b-bf76-087f85098c9b
|
| 49 | 2026-05-14T16:08:36.270Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_63d1fcbf-1b29-47c8-a6d4-05e3a30623cc/crucible/regions/f33a8e81-b1a1-4f0b-bf76-087f85098c9b@d9acf02d-46bc-424f-a213-37939f99a0a0
|
| 50 | 2026-05-14T16:08:36.270Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_63d1fcbf-1b29-47c8-a6d4-05e3a30623cc/crucible/regions/f33a8e81-b1a1-4f0b-bf76-087f85098c9b", "d9acf02d-46bc-424f-a213-37939f99a0a0"]
|
| 51 | 2026-05-14T16:08:36.276Z | INFO | crucible-agent (dropshot): request completed
latency_us = 19180
local_addr = [fd00:1122:3344:101::1a]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:44389
req_id = 942fd98e-a24e-4a88-8e98-1813c66bb1bc
response_code = 200
uri = /crucible/0/regions/f33a8e81-b1a1-4f0b-bf76-087f85098c9b/snapshots/d9acf02d-46bc-424f-a213-37939f99a0a0/run
|
| 52 | 2026-05-14T16:08:40.743Z | INFO | crucible-agent (datafile): region 109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9 state: Requested
|
| 53 | 2026-05-14T16:08:40.743Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 54 | 2026-05-14T16:08:40.743Z | INFO | crucible-agent (dropshot): request completed
latency_us = 240
local_addr = [fd00:1122:3344:101::1a]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:44389
req_id = 0dc4a8eb-9490-4e65-bfbc-04500a76a2b5
response_code = 200
uri = /crucible/0/regions
|
| 55 | 2026-05-14T16:08:40.750Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_63d1fcbf-1b29-47c8-a6d4-05e3a30623cc/crucible/regions/109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9
|
| 56 | 2026-05-14T16:08:40.750Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_63d1fcbf-1b29-47c8-a6d4-05e3a30623cc/crucible/regions/109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9
|
| 57 | 2026-05-14T16:08:40.788Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9"
region = 109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9
|
| 58 | 2026-05-14T16:08:40.788Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9" "--data" "/data/regions/109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = 109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9
|
| 59 | 2026-05-14T16:08:40.837Z | INFO | crucible-agent (worker): region files created ok
region = 109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9
|
| 60 | 2026-05-14T16:08:40.837Z | INFO | crucible-agent (datafile): region 109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9 state: Requested -> Created
|
| 61 | 2026-05-14T16:08:40.837Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 62 | 2026-05-14T16:08:40.838Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9
|
| 63 | 2026-05-14T16:08:40.840Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9
|
| 64 | 2026-05-14T16:08:40.843Z | INFO | crucible-agent (worker): creating config property group
|
| 65 | 2026-05-14T16:08:40.844Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9
|
| 66 | 2026-05-14T16:08:40.844Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9
|
| 67 | 2026-05-14T16:08:40.844Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19003
|
| 68 | 2026-05-14T16:08:40.844Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a
|
| 69 | 2026-05-14T16:08:40.845Z | INFO | crucible-agent (worker): commit
|
| 70 | 2026-05-14T16:08:40.847Z | INFO | crucible-agent (worker): ok!
|
| 71 | 2026-05-14T16:08:40.853Z | INFO | crucible-agent (worker): SMF ok!
|
| 72 | 2026-05-14T16:08:41.047Z | INFO | crucible-agent (dropshot): request completed
latency_us = 57
local_addr = [fd00:1122:3344:101::1a]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:44389
req_id = 967f70b2-8427-4b1f-9bde-37c782a7dbfa
response_code = 200
uri = /crucible/0/regions
|
| 73 | 2026-05-14T16:09:05.468Z | INFO | crucible-agent (dropshot): request completed
latency_us = 51
local_addr = [fd00:1122:3344:101::1a]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:44389
req_id = 13e0ef0a-3c60-4a68-a77a-89dc05b451e0
response_code = 200
uri = /crucible/0/regions/109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9
|
| 74 | 2026-05-14T16:09:05.472Z | INFO | crucible-agent (datafile): path is "/data/regions/109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9"
|
| 75 | 2026-05-14T16:09:05.483Z | INFO | crucible-agent (datafile): dataset is oxp_63d1fcbf-1b29-47c8-a6d4-05e3a30623cc/crucible/regions/109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9
|
| 76 | 2026-05-14T16:09:05.490Z | INFO | crucible-agent (datafile): region 109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9 state: Created -> Tombstoned
|
| 77 | 2026-05-14T16:09:05.490Z | INFO | crucible-agent (worker): applying SMF actions before removal...
|
| 78 | 2026-05-14T16:09:05.490Z | INFO | crucible-agent (dropshot): request completed
latency_us = 18042
local_addr = [fd00:1122:3344:101::1a]:32345
method = DELETE
remote_addr = [fd00:1122:3344:101::a]:44389
req_id = 9c8f8eb8-1b3d-492b-b93b-3ffd88e032e0
response_code = 204
uri = /crucible/0/regions/109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9
|
| 79 | 2026-05-14T16:09:05.492Z | INFO | crucible-agent (worker): disabling downstairs instance: downstairs-109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9 (instance states: (Some(Online), None))
|
| 80 | 2026-05-14T16:09:05.495Z | INFO | crucible-agent (dropshot): request completed
latency_us = 47
local_addr = [fd00:1122:3344:101::1a]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:44389
req_id = 72ad2056-8ccc-4c59-9974-d43bf90ed94d
response_code = 200
uri = /crucible/0/regions/109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9
|
| 81 | 2026-05-14T16:09:05.497Z | INFO | crucible-agent (worker): SMF ok!
|
| 82 | 2026-05-14T16:09:05.506Z | INFO | crucible-agent (worker): deleting zfs dataset "oxp_63d1fcbf-1b29-47c8-a6d4-05e3a30623cc/crucible/regions/109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9"
region = 109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9
|
| 83 | 2026-05-14T16:09:05.558Z | INFO | crucible-agent (datafile): region 109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9 state: Tombstoned -> Destroyed
|
| 84 | 2026-05-14T16:09:05.579Z | INFO | crucible-agent (dropshot): request completed
latency_us = 52
local_addr = [fd00:1122:3344:101::1a]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:44389
req_id = 5129e657-88b7-4336-bee7-e24fdb9b5a31
response_code = 200
uri = /crucible/0/regions/109b7b35-e8d7-4f1f-8ead-6e9bc0e6fcf9
|