| 1 | 2026-03-02T04:32:16.110Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::13]:32345
remote_addr = [fd00:1122:3344:101::a]:37344
|
| 2 | 2026-03-02T04:32:16.110Z | INFO | crucible-agent (datafile): region f8c46a57-6727-453e-a650-5432261a159f state: Requested
|
| 3 | 2026-03-02T04:32:16.111Z | INFO | crucible-agent (dropshot): request completed
latency_us = 671
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:37344
req_id = 369f9b41-bbc3-4e0e-b156-f71636d2d175
response_code = 200
uri = /crucible/0/regions
|
| 4 | 2026-03-02T04:32:16.111Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 5 | 2026-03-02T04:32:16.120Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/f8c46a57-6727-453e-a650-5432261a159f
|
| 6 | 2026-03-02T04:32:16.120Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/f8c46a57-6727-453e-a650-5432261a159f
|
| 7 | 2026-03-02T04:32:16.175Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("f8c46a57-6727-453e-a650-5432261a159f"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/f8c46a57-6727-453e-a650-5432261a159f"
region = f8c46a57-6727-453e-a650-5432261a159f
|
| 8 | 2026-03-02T04:32:16.175Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "f8c46a57-6727-453e-a650-5432261a159f" "--data" "/data/regions/f8c46a57-6727-453e-a650-5432261a159f" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = f8c46a57-6727-453e-a650-5432261a159f
|
| 9 | 2026-03-02T04:32:16.256Z | INFO | crucible-agent (worker): region files created ok
region = f8c46a57-6727-453e-a650-5432261a159f
|
| 10 | 2026-03-02T04:32:16.256Z | INFO | crucible-agent (datafile): region f8c46a57-6727-453e-a650-5432261a159f state: Requested -> Created
|
| 11 | 2026-03-02T04:32:16.256Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 12 | 2026-03-02T04:32:16.278Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-f8c46a57-6727-453e-a650-5432261a159f
|
| 13 | 2026-03-02T04:32:16.290Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-f8c46a57-6727-453e-a650-5432261a159f
|
| 14 | 2026-03-02T04:32:16.299Z | INFO | crucible-agent (worker): creating config property group
|
| 15 | 2026-03-02T04:32:16.303Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-f8c46a57-6727-453e-a650-5432261a159f
|
| 16 | 2026-03-02T04:32:16.303Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/f8c46a57-6727-453e-a650-5432261a159f
|
| 17 | 2026-03-02T04:32:16.303Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19001
|
| 18 | 2026-03-02T04:32:16.303Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13
|
| 19 | 2026-03-02T04:32:16.303Z | INFO | crucible-agent (worker): commit
|
| 20 | 2026-03-02T04:32:16.307Z | INFO | crucible-agent (worker): ok!
|
| 21 | 2026-03-02T04:32:16.311Z | INFO | crucible-agent (worker): SMF ok!
|
| 22 | 2026-03-02T04:32:16.480Z | INFO | crucible-agent (dropshot): request completed
latency_us = 103
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:37344
req_id = 4248832f-c284-4940-967c-ca6ac190a6c7
response_code = 200
uri = /crucible/0/regions
|
| 23 | 2026-03-02T04:32:18.127Z | INFO | crucible-agent (dropshot): request completed
latency_us = 54
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:37344
req_id = 57bad9f1-7593-43a9-be4d-482b53804ec2
response_code = 200
uri = /crucible/0/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca
|
| 24 | 2026-03-02T04:32:18.130Z | INFO | crucible-agent (datafile): path is "/data/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca"
|
| 25 | 2026-03-02T04:32:18.136Z | INFO | crucible-agent (datafile): dataset is oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca
|
| 26 | 2026-03-02T04:32:18.143Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca@52e7893d-f554-4c3e-906e-1d06fca48dc8
|
| 27 | 2026-03-02T04:32:18.143Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca", "52e7893d-f554-4c3e-906e-1d06fca48dc8"]
|
| 28 | 2026-03-02T04:32:18.150Z | INFO | crucible-agent (dropshot): request completed
latency_us = 20036
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:37344
req_id = ce3545fc-1d05-4881-b473-831fe2cc4423
response_code = 200
uri = /crucible/0/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca/snapshots/52e7893d-f554-4c3e-906e-1d06fca48dc8
|
| 29 | 2026-03-02T04:32:18.154Z | INFO | crucible-agent (datafile): path is "/data/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca"
|
| 30 | 2026-03-02T04:32:18.161Z | INFO | crucible-agent (datafile): dataset is oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca
|
| 31 | 2026-03-02T04:32:18.167Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca@52e7893d-f554-4c3e-906e-1d06fca48dc8
|
| 32 | 2026-03-02T04:32:18.167Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca", "52e7893d-f554-4c3e-906e-1d06fca48dc8"]
|
| 33 | 2026-03-02T04:32:18.173Z | INFO | crucible-agent (datafile): requesting running snapshot 528b89cc-82c2-453d-8a03-39a2394fe2ca-52e7893d-f554-4c3e-906e-1d06fca48dc8 state: Requested
|
| 34 | 2026-03-02T04:32:18.174Z | INFO | crucible-agent (dropshot): request completed
latency_us = 19776
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:37344
req_id = 2be05ce8-1228-460d-8f95-1a655b4c7dce
response_code = 200
uri = /crucible/0/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca/snapshots/52e7893d-f554-4c3e-906e-1d06fca48dc8/run
|
| 35 | 2026-03-02T04:32:18.174Z | INFO | crucible-agent (worker): applying SMF actions for region 528b89cc-82c2-453d-8a03-39a2394fe2ca running snapshot 52e7893d-f554-4c3e-906e-1d06fca48dc8 (state Requested)...
|
| 36 | 2026-03-02T04:32:18.179Z | INFO | crucible-agent (worker): creating missing snapshot instance snapshot-528b89cc-82c2-453d-8a03-39a2394fe2ca-52e7893d-f554-4c3e-906e-1d06fca48dc8
|
| 37 | 2026-03-02T04:32:18.186Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:snapshot-528b89cc-82c2-453d-8a03-39a2394fe2ca-52e7893d-f554-4c3e-906e-1d06fca48dc8
|
| 38 | 2026-03-02T04:32:18.195Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:snapshot-528b89cc-82c2-453d-8a03-39a2394fe2ca-52e7893d-f554-4c3e-906e-1d06fca48dc8
|
| 39 | 2026-03-02T04:32:18.195Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca/.zfs/snapshot/52e7893d-f554-4c3e-906e-1d06fca48dc8
|
| 40 | 2026-03-02T04:32:18.195Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19002
|
| 41 | 2026-03-02T04:32:18.195Z | INFO | crucible-agent (worker): ensure mode SCF_TYPE_ASTRING ro
|
| 42 | 2026-03-02T04:32:18.195Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13
|
| 43 | 2026-03-02T04:32:18.195Z | INFO | crucible-agent (worker): commit
|
| 44 | 2026-03-02T04:32:18.199Z | INFO | crucible-agent (worker): ok!
|
| 45 | 2026-03-02T04:32:18.204Z | INFO | crucible-agent (worker): SMF ok!
|
| 46 | 2026-03-02T04:32:18.204Z | INFO | crucible-agent (datafile): region 528b89cc-82c2-453d-8a03-39a2394fe2ca running snapshot 52e7893d-f554-4c3e-906e-1d06fca48dc8 state: Requested -> Created
|
| 47 | 2026-03-02T04:32:18.543Z | INFO | crucible-agent (datafile): path is "/data/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca"
|
| 48 | 2026-03-02T04:32:18.555Z | INFO | crucible-agent (datafile): dataset is oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca
|
| 49 | 2026-03-02T04:32:18.564Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca@52e7893d-f554-4c3e-906e-1d06fca48dc8
|
| 50 | 2026-03-02T04:32:18.564Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca", "52e7893d-f554-4c3e-906e-1d06fca48dc8"]
|
| 51 | 2026-03-02T04:32:18.573Z | INFO | crucible-agent (dropshot): request completed
latency_us = 30067
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:37344
req_id = 1a747200-d8b2-4994-9610-913c25bf9d0d
response_code = 200
uri = /crucible/0/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca/snapshots/52e7893d-f554-4c3e-906e-1d06fca48dc8/run
|
| 52 | 2026-03-02T04:32:23.304Z | INFO | crucible-agent (datafile): region a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee state: Requested
|
| 53 | 2026-03-02T04:32:23.304Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 54 | 2026-03-02T04:32:23.304Z | INFO | crucible-agent (dropshot): request completed
latency_us = 221
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:37344
req_id = e331ee06-0deb-4bd9-b5d9-558425385fd1
response_code = 200
uri = /crucible/0/regions
|
| 55 | 2026-03-02T04:32:23.311Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee
|
| 56 | 2026-03-02T04:32:23.311Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee
|
| 57 | 2026-03-02T04:32:23.358Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee"
region = a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee
|
| 58 | 2026-03-02T04:32:23.358Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee" "--data" "/data/regions/a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee
|
| 59 | 2026-03-02T04:32:23.431Z | INFO | crucible-agent (worker): region files created ok
region = a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee
|
| 60 | 2026-03-02T04:32:23.431Z | INFO | crucible-agent (datafile): region a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee state: Requested -> Created
|
| 61 | 2026-03-02T04:32:23.431Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 62 | 2026-03-02T04:32:23.432Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee
|
| 63 | 2026-03-02T04:32:23.435Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee
|
| 64 | 2026-03-02T04:32:23.442Z | INFO | crucible-agent (dropshot): request completed
latency_us = 53
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:37344
req_id = 03eaf13e-9ea3-4fad-9a01-4f8f6e97b3f1
response_code = 200
uri = /crucible/0/regions
|
| 65 | 2026-03-02T04:32:23.442Z | INFO | crucible-agent (worker): creating config property group
|
| 66 | 2026-03-02T04:32:23.445Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee
|
| 67 | 2026-03-02T04:32:23.445Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee
|
| 68 | 2026-03-02T04:32:23.445Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19003
|
| 69 | 2026-03-02T04:32:23.445Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13
|
| 70 | 2026-03-02T04:32:23.445Z | INFO | crucible-agent (worker): commit
|
| 71 | 2026-03-02T04:32:23.451Z | INFO | crucible-agent (worker): ok!
|
| 72 | 2026-03-02T04:32:23.461Z | INFO | crucible-agent (worker): SMF ok!
|
| 73 | 2026-03-02T04:33:00.492Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::13]:32345
remote_addr = [fd00:1122:3344:101::a]:61238
|
| 74 | 2026-03-02T04:33:00.492Z | INFO | crucible-agent (dropshot): request completed
latency_us = 40
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:61238
req_id = 1cdb8705-44cd-4f19-a34e-d1c27d6152e4
response_code = 200
uri = /crucible/0/regions/a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee
|
| 75 | 2026-03-02T04:33:00.495Z | INFO | crucible-agent (datafile): path is "/data/regions/a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee"
|
| 76 | 2026-03-02T04:33:00.505Z | INFO | crucible-agent (datafile): dataset is oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee
|
| 77 | 2026-03-02T04:33:00.512Z | INFO | crucible-agent (datafile): region a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee state: Created -> Tombstoned
|
| 78 | 2026-03-02T04:33:00.512Z | INFO | crucible-agent (dropshot): request completed
latency_us = 17901
local_addr = [fd00:1122:3344:101::13]:32345
method = DELETE
remote_addr = [fd00:1122:3344:101::a]:61238
req_id = 7c511f51-0fd3-4986-b56f-fe38551e59fc
response_code = 204
uri = /crucible/0/regions/a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee
|
| 79 | 2026-03-02T04:33:00.512Z | INFO | crucible-agent (worker): applying SMF actions before removal...
|
| 80 | 2026-03-02T04:33:00.513Z | INFO | crucible-agent (worker): disabling downstairs instance: downstairs-a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee (instance states: (Some(Online), None))
|
| 81 | 2026-03-02T04:33:00.516Z | INFO | crucible-agent (dropshot): request completed
latency_us = 105
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:61238
req_id = 244457f5-35d7-441f-943f-172f4f7a137d
response_code = 200
uri = /crucible/0/regions/a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee
|
| 82 | 2026-03-02T04:33:00.529Z | INFO | crucible-agent (worker): SMF ok!
|
| 83 | 2026-03-02T04:33:00.538Z | INFO | crucible-agent (worker): deleting zfs dataset "oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee"
region = a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee
|
| 84 | 2026-03-02T04:33:00.594Z | INFO | crucible-agent (datafile): region a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee state: Tombstoned -> Destroyed
|
| 85 | 2026-03-02T04:33:00.640Z | INFO | crucible-agent (dropshot): request completed
latency_us = 62
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:61238
req_id = cecfb41c-9f28-4b71-98cb-3c30cd4f3652
response_code = 200
uri = /crucible/0/regions/a1bb6ba2-a42d-4b61-ad9e-324b0072f3ee
|