| 1 | 2026-01-05T19:19:07.660Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::18]:32345
remote_addr = [fd00:1122:3344:101::b]:53642
|
| 2 | 2026-01-05T19:19:07.660Z | INFO | crucible-agent (datafile): region 3883c05d-d915-4721-9fe4-2e25c4066490 state: Requested
|
| 3 | 2026-01-05T19:19:07.662Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 4 | 2026-01-05T19:19:07.662Z | INFO | crucible-agent (dropshot): request completed
latency_us = 1128
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:53642
req_id = 005055ef-0d6c-4a2e-b916-34a6df08d4f5
response_code = 200
uri = /crucible/0/regions
|
| 5 | 2026-01-05T19:19:07.671Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_79d482c9-d71a-455a-b826-48138a60ace1/crucible/regions/3883c05d-d915-4721-9fe4-2e25c4066490
|
| 6 | 2026-01-05T19:19:07.672Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_79d482c9-d71a-455a-b826-48138a60ace1/crucible/regions/3883c05d-d915-4721-9fe4-2e25c4066490
|
| 7 | 2026-01-05T19:19:07.731Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("3883c05d-d915-4721-9fe4-2e25c4066490"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/3883c05d-d915-4721-9fe4-2e25c4066490"
region = 3883c05d-d915-4721-9fe4-2e25c4066490
|
| 8 | 2026-01-05T19:19:07.732Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "3883c05d-d915-4721-9fe4-2e25c4066490" "--data" "/data/regions/3883c05d-d915-4721-9fe4-2e25c4066490" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = 3883c05d-d915-4721-9fe4-2e25c4066490
|
| 9 | 2026-01-05T19:19:07.866Z | INFO | crucible-agent (worker): region files created ok
region = 3883c05d-d915-4721-9fe4-2e25c4066490
|
| 10 | 2026-01-05T19:19:07.866Z | INFO | crucible-agent (datafile): region 3883c05d-d915-4721-9fe4-2e25c4066490 state: Requested -> Created
|
| 11 | 2026-01-05T19:19:07.866Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 12 | 2026-01-05T19:19:07.867Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-3883c05d-d915-4721-9fe4-2e25c4066490
|
| 13 | 2026-01-05T19:19:07.877Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-3883c05d-d915-4721-9fe4-2e25c4066490
|
| 14 | 2026-01-05T19:19:07.905Z | INFO | crucible-agent (worker): creating config property group
|
| 15 | 2026-01-05T19:19:07.911Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-3883c05d-d915-4721-9fe4-2e25c4066490
|
| 16 | 2026-01-05T19:19:07.911Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/3883c05d-d915-4721-9fe4-2e25c4066490
|
| 17 | 2026-01-05T19:19:07.911Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19001
|
| 18 | 2026-01-05T19:19:07.911Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18
|
| 19 | 2026-01-05T19:19:07.911Z | INFO | crucible-agent (worker): commit
|
| 20 | 2026-01-05T19:19:07.915Z | INFO | crucible-agent (worker): ok!
|
| 21 | 2026-01-05T19:19:07.932Z | INFO | crucible-agent (worker): SMF ok!
|
| 22 | 2026-01-05T19:19:07.955Z | INFO | crucible-agent (dropshot): request completed
latency_us = 63
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:53642
req_id = 07bdffae-49ba-4377-9d3a-364b9d3fb414
response_code = 200
uri = /crucible/0/regions
|
| 23 | 2026-01-05T19:19:09.183Z | INFO | crucible-agent (dropshot): request completed
latency_us = 69
local_addr = [fd00:1122:3344:101::18]:32345
method = GET
remote_addr = [fd00:1122:3344:101::b]:53642
req_id = c58eb327-bc86-487c-b9bc-7e6889f37bd2
response_code = 200
uri = /crucible/0/regions/83e57d2d-6806-417c-8803-2c34078fe3a7
|
| 24 | 2026-01-05T19:19:09.186Z | INFO | crucible-agent (datafile): path is "/data/regions/83e57d2d-6806-417c-8803-2c34078fe3a7"
|
| 25 | 2026-01-05T19:19:09.192Z | INFO | crucible-agent (datafile): dataset is oxp_79d482c9-d71a-455a-b826-48138a60ace1/crucible/regions/83e57d2d-6806-417c-8803-2c34078fe3a7
|
| 26 | 2026-01-05T19:19:09.198Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_79d482c9-d71a-455a-b826-48138a60ace1/crucible/regions/83e57d2d-6806-417c-8803-2c34078fe3a7@de6948a4-a1ea-4aa9-a0bf-2df0073e7a05
|
| 27 | 2026-01-05T19:19:09.198Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_79d482c9-d71a-455a-b826-48138a60ace1/crucible/regions/83e57d2d-6806-417c-8803-2c34078fe3a7", "de6948a4-a1ea-4aa9-a0bf-2df0073e7a05"]
|
| 28 | 2026-01-05T19:19:09.207Z | INFO | crucible-agent (dropshot): request completed
latency_us = 20935
local_addr = [fd00:1122:3344:101::18]:32345
method = GET
remote_addr = [fd00:1122:3344:101::b]:53642
req_id = f3044be5-7552-4375-9b19-17ccbc5b2672
response_code = 200
uri = /crucible/0/regions/83e57d2d-6806-417c-8803-2c34078fe3a7/snapshots/de6948a4-a1ea-4aa9-a0bf-2df0073e7a05
|
| 29 | 2026-01-05T19:19:09.211Z | INFO | crucible-agent (datafile): path is "/data/regions/83e57d2d-6806-417c-8803-2c34078fe3a7"
|
| 30 | 2026-01-05T19:19:09.216Z | INFO | crucible-agent (datafile): dataset is oxp_79d482c9-d71a-455a-b826-48138a60ace1/crucible/regions/83e57d2d-6806-417c-8803-2c34078fe3a7
|
| 31 | 2026-01-05T19:19:09.221Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_79d482c9-d71a-455a-b826-48138a60ace1/crucible/regions/83e57d2d-6806-417c-8803-2c34078fe3a7@de6948a4-a1ea-4aa9-a0bf-2df0073e7a05
|
| 32 | 2026-01-05T19:19:09.221Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_79d482c9-d71a-455a-b826-48138a60ace1/crucible/regions/83e57d2d-6806-417c-8803-2c34078fe3a7", "de6948a4-a1ea-4aa9-a0bf-2df0073e7a05"]
|
| 33 | 2026-01-05T19:19:09.228Z | INFO | crucible-agent (datafile): requesting running snapshot 83e57d2d-6806-417c-8803-2c34078fe3a7-de6948a4-a1ea-4aa9-a0bf-2df0073e7a05 state: Requested
|
| 34 | 2026-01-05T19:19:09.228Z | INFO | crucible-agent (dropshot): request completed
latency_us = 17396
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:53642
req_id = afdb77ac-4a04-4765-9216-8359c2533183
response_code = 200
uri = /crucible/0/regions/83e57d2d-6806-417c-8803-2c34078fe3a7/snapshots/de6948a4-a1ea-4aa9-a0bf-2df0073e7a05/run
|
| 35 | 2026-01-05T19:19:09.228Z | INFO | crucible-agent (worker): applying SMF actions for region 83e57d2d-6806-417c-8803-2c34078fe3a7 running snapshot de6948a4-a1ea-4aa9-a0bf-2df0073e7a05 (state Requested)...
|
| 36 | 2026-01-05T19:19:09.233Z | INFO | crucible-agent (worker): creating missing snapshot instance snapshot-83e57d2d-6806-417c-8803-2c34078fe3a7-de6948a4-a1ea-4aa9-a0bf-2df0073e7a05
|
| 37 | 2026-01-05T19:19:09.242Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:snapshot-83e57d2d-6806-417c-8803-2c34078fe3a7-de6948a4-a1ea-4aa9-a0bf-2df0073e7a05
|
| 38 | 2026-01-05T19:19:09.252Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:snapshot-83e57d2d-6806-417c-8803-2c34078fe3a7-de6948a4-a1ea-4aa9-a0bf-2df0073e7a05
|
| 39 | 2026-01-05T19:19:09.252Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/83e57d2d-6806-417c-8803-2c34078fe3a7/.zfs/snapshot/de6948a4-a1ea-4aa9-a0bf-2df0073e7a05
|
| 40 | 2026-01-05T19:19:09.252Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19002
|
| 41 | 2026-01-05T19:19:09.252Z | INFO | crucible-agent (worker): ensure mode SCF_TYPE_ASTRING ro
|
| 42 | 2026-01-05T19:19:09.252Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18
|
| 43 | 2026-01-05T19:19:09.252Z | INFO | crucible-agent (worker): commit
|
| 44 | 2026-01-05T19:19:09.256Z | INFO | crucible-agent (worker): ok!
|
| 45 | 2026-01-05T19:19:09.260Z | INFO | crucible-agent (worker): SMF ok!
|
| 46 | 2026-01-05T19:19:09.260Z | INFO | crucible-agent (datafile): region 83e57d2d-6806-417c-8803-2c34078fe3a7 running snapshot de6948a4-a1ea-4aa9-a0bf-2df0073e7a05 state: Requested -> Created
|
| 47 | 2026-01-05T19:19:09.472Z | INFO | crucible-agent (datafile): path is "/data/regions/83e57d2d-6806-417c-8803-2c34078fe3a7"
|
| 48 | 2026-01-05T19:19:09.479Z | INFO | crucible-agent (datafile): dataset is oxp_79d482c9-d71a-455a-b826-48138a60ace1/crucible/regions/83e57d2d-6806-417c-8803-2c34078fe3a7
|
| 49 | 2026-01-05T19:19:09.486Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_79d482c9-d71a-455a-b826-48138a60ace1/crucible/regions/83e57d2d-6806-417c-8803-2c34078fe3a7@de6948a4-a1ea-4aa9-a0bf-2df0073e7a05
|
| 50 | 2026-01-05T19:19:09.486Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_79d482c9-d71a-455a-b826-48138a60ace1/crucible/regions/83e57d2d-6806-417c-8803-2c34078fe3a7", "de6948a4-a1ea-4aa9-a0bf-2df0073e7a05"]
|
| 51 | 2026-01-05T19:19:09.491Z | INFO | crucible-agent (dropshot): request completed
latency_us = 18654
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:53642
req_id = 644327f2-42ba-42be-8693-ec481d484fb0
response_code = 200
uri = /crucible/0/regions/83e57d2d-6806-417c-8803-2c34078fe3a7/snapshots/de6948a4-a1ea-4aa9-a0bf-2df0073e7a05/run
|
| 52 | 2026-01-05T19:19:14.365Z | INFO | crucible-agent (datafile): region f3a167a0-643d-409e-962d-26edaa10cd3c state: Requested
|
| 53 | 2026-01-05T19:19:14.365Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 54 | 2026-01-05T19:19:14.365Z | INFO | crucible-agent (dropshot): request completed
latency_us = 252
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:53642
req_id = fcf6ce42-a0a5-49e3-a282-eba1b0f873dc
response_code = 200
uri = /crucible/0/regions
|
| 55 | 2026-01-05T19:19:14.372Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_79d482c9-d71a-455a-b826-48138a60ace1/crucible/regions/f3a167a0-643d-409e-962d-26edaa10cd3c
|
| 56 | 2026-01-05T19:19:14.372Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_79d482c9-d71a-455a-b826-48138a60ace1/crucible/regions/f3a167a0-643d-409e-962d-26edaa10cd3c
|
| 57 | 2026-01-05T19:19:14.443Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("f3a167a0-643d-409e-962d-26edaa10cd3c"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/f3a167a0-643d-409e-962d-26edaa10cd3c"
region = f3a167a0-643d-409e-962d-26edaa10cd3c
|
| 58 | 2026-01-05T19:19:14.443Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "f3a167a0-643d-409e-962d-26edaa10cd3c" "--data" "/data/regions/f3a167a0-643d-409e-962d-26edaa10cd3c" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = f3a167a0-643d-409e-962d-26edaa10cd3c
|
| 59 | 2026-01-05T19:19:14.522Z | INFO | crucible-agent (worker): region files created ok
region = f3a167a0-643d-409e-962d-26edaa10cd3c
|
| 60 | 2026-01-05T19:19:14.522Z | INFO | crucible-agent (datafile): region f3a167a0-643d-409e-962d-26edaa10cd3c state: Requested -> Created
|
| 61 | 2026-01-05T19:19:14.522Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 62 | 2026-01-05T19:19:14.524Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-f3a167a0-643d-409e-962d-26edaa10cd3c
|
| 63 | 2026-01-05T19:19:14.527Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-f3a167a0-643d-409e-962d-26edaa10cd3c
|
| 64 | 2026-01-05T19:19:14.535Z | INFO | crucible-agent (worker): creating config property group
|
| 65 | 2026-01-05T19:19:14.538Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-f3a167a0-643d-409e-962d-26edaa10cd3c
|
| 66 | 2026-01-05T19:19:14.539Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/f3a167a0-643d-409e-962d-26edaa10cd3c
|
| 67 | 2026-01-05T19:19:14.539Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19003
|
| 68 | 2026-01-05T19:19:14.539Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18
|
| 69 | 2026-01-05T19:19:14.539Z | INFO | crucible-agent (worker): commit
|
| 70 | 2026-01-05T19:19:14.542Z | INFO | crucible-agent (worker): ok!
|
| 71 | 2026-01-05T19:19:14.548Z | INFO | crucible-agent (worker): SMF ok!
|
| 72 | 2026-01-05T19:19:14.704Z | INFO | crucible-agent (dropshot): request completed
latency_us = 63
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:53642
req_id = c0a71c00-5c3c-44a1-bbf2-bfe6cef50c6d
response_code = 200
uri = /crucible/0/regions
|
| 73 | 2026-01-05T19:19:41.720Z | INFO | crucible-agent (dropshot): request completed
latency_us = 71
local_addr = [fd00:1122:3344:101::18]:32345
method = GET
remote_addr = [fd00:1122:3344:101::b]:53642
req_id = 2ab01d48-06c4-4ab1-9eb5-39d9fd08e097
response_code = 200
uri = /crucible/0/regions/f3a167a0-643d-409e-962d-26edaa10cd3c
|
| 74 | 2026-01-05T19:19:41.723Z | INFO | crucible-agent (datafile): path is "/data/regions/f3a167a0-643d-409e-962d-26edaa10cd3c"
|
| 75 | 2026-01-05T19:19:41.735Z | INFO | crucible-agent (datafile): dataset is oxp_79d482c9-d71a-455a-b826-48138a60ace1/crucible/regions/f3a167a0-643d-409e-962d-26edaa10cd3c
|
| 76 | 2026-01-05T19:19:41.741Z | INFO | crucible-agent (datafile): region f3a167a0-643d-409e-962d-26edaa10cd3c state: Created -> Tombstoned
|
| 77 | 2026-01-05T19:19:41.742Z | INFO | crucible-agent (worker): applying SMF actions before removal...
|
| 78 | 2026-01-05T19:19:41.742Z | INFO | crucible-agent (dropshot): request completed
latency_us = 18923
local_addr = [fd00:1122:3344:101::18]:32345
method = DELETE
remote_addr = [fd00:1122:3344:101::b]:53642
req_id = c7d87487-4585-47e3-91f1-2cc9e58f55ab
response_code = 204
uri = /crucible/0/regions/f3a167a0-643d-409e-962d-26edaa10cd3c
|
| 79 | 2026-01-05T19:19:41.743Z | INFO | crucible-agent (worker): disabling downstairs instance: downstairs-f3a167a0-643d-409e-962d-26edaa10cd3c (instance states: (Some(Online), None))
|
| 80 | 2026-01-05T19:19:41.745Z | INFO | crucible-agent (dropshot): request completed
latency_us = 30
local_addr = [fd00:1122:3344:101::18]:32345
method = GET
remote_addr = [fd00:1122:3344:101::b]:53642
req_id = c97a103d-8e9b-477b-b84c-aa15b13a2a46
response_code = 200
uri = /crucible/0/regions/f3a167a0-643d-409e-962d-26edaa10cd3c
|
| 81 | 2026-01-05T19:19:41.757Z | INFO | crucible-agent (worker): SMF ok!
|
| 82 | 2026-01-05T19:19:41.765Z | INFO | crucible-agent (worker): deleting zfs dataset "oxp_79d482c9-d71a-455a-b826-48138a60ace1/crucible/regions/f3a167a0-643d-409e-962d-26edaa10cd3c"
region = f3a167a0-643d-409e-962d-26edaa10cd3c
|
| 83 | 2026-01-05T19:19:41.825Z | INFO | crucible-agent (datafile): region f3a167a0-643d-409e-962d-26edaa10cd3c state: Tombstoned -> Destroyed
|
| 84 | 2026-01-05T19:19:41.837Z | INFO | crucible-agent (dropshot): request completed
latency_us = 50
local_addr = [fd00:1122:3344:101::18]:32345
method = GET
remote_addr = [fd00:1122:3344:101::b]:53642
req_id = 864f85b0-a76f-4f08-9a54-7606ad75f226
response_code = 200
uri = /crucible/0/regions/f3a167a0-643d-409e-962d-26edaa10cd3c
|