| 1 | 2026-08-25T21:03:54.225Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::14]:32345
remote_addr = [fd00:1122:3344:101::b]:40517
|
| 2 | 2026-08-25T21:03:54.225Z | INFO | crucible-agent (datafile): region 17a0e9d3-db2b-450b-a785-bdbef796a723 state: Requested
|
| 3 | 2026-08-25T21:03:54.226Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 4 | 2026-08-25T21:03:54.226Z | INFO | crucible-agent (dropshot): request completed
latency_us = 682
local_addr = [fd00:1122:3344:101::14]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:40517
req_id = f4448d5e-556d-420b-8490-37a44291e6c4
response_code = 200
uri = /crucible/0/regions
|
| 5 | 2026-08-25T21:03:54.245Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_a7733fa5-3cc8-42fc-829c-28163a3731c1/crucible/regions/17a0e9d3-db2b-450b-a785-bdbef796a723
|
| 6 | 2026-08-25T21:03:54.245Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_a7733fa5-3cc8-42fc-829c-28163a3731c1/crucible/regions/17a0e9d3-db2b-450b-a785-bdbef796a723
|
| 7 | 2026-08-25T21:03:54.484Z | INFO | crucible-agent (dropshot): request completed
latency_us = 61
local_addr = [fd00:1122:3344:101::14]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:40517
req_id = 159e5439-54ed-4064-abd2-7f8fb3fa7955
response_code = 200
uri = /crucible/0/regions
|
| 8 | 2026-08-25T21:03:55.060Z | INFO | crucible-agent (dropshot): request completed
latency_us = 58
local_addr = [fd00:1122:3344:101::14]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:40517
req_id = c3139aeb-f273-4058-8955-9ec1947ef21d
response_code = 200
uri = /crucible/0/regions
|
| 9 | 2026-08-25T21:03:55.130Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("17a0e9d3-db2b-450b-a785-bdbef796a723"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/17a0e9d3-db2b-450b-a785-bdbef796a723"
region = 17a0e9d3-db2b-450b-a785-bdbef796a723
|
| 10 | 2026-08-25T21:03:55.130Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "17a0e9d3-db2b-450b-a785-bdbef796a723" "--data" "/data/regions/17a0e9d3-db2b-450b-a785-bdbef796a723" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = 17a0e9d3-db2b-450b-a785-bdbef796a723
|
| 11 | 2026-08-25T21:03:56.149Z | INFO | crucible-agent (dropshot): request completed
latency_us = 56
local_addr = [fd00:1122:3344:101::14]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:40517
req_id = 15112268-d024-440f-bf36-310e33b0aebb
response_code = 200
uri = /crucible/0/regions
|
| 12 | 2026-08-25T21:03:56.522Z | INFO | crucible-agent (worker): region files created ok
region = 17a0e9d3-db2b-450b-a785-bdbef796a723
|
| 13 | 2026-08-25T21:03:56.522Z | INFO | crucible-agent (datafile): region 17a0e9d3-db2b-450b-a785-bdbef796a723 state: Requested -> Created
|
| 14 | 2026-08-25T21:03:56.522Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 15 | 2026-08-25T21:03:56.522Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-17a0e9d3-db2b-450b-a785-bdbef796a723
|
| 16 | 2026-08-25T21:03:56.536Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-17a0e9d3-db2b-450b-a785-bdbef796a723
|
| 17 | 2026-08-25T21:03:56.580Z | INFO | crucible-agent (worker): creating config property group
|
| 18 | 2026-08-25T21:03:56.592Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-17a0e9d3-db2b-450b-a785-bdbef796a723
|
| 19 | 2026-08-25T21:03:56.593Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/17a0e9d3-db2b-450b-a785-bdbef796a723
|
| 20 | 2026-08-25T21:03:56.593Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19001
|
| 21 | 2026-08-25T21:03:56.593Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14
|
| 22 | 2026-08-25T21:03:56.593Z | INFO | crucible-agent (worker): commit
|
| 23 | 2026-08-25T21:03:56.606Z | INFO | crucible-agent (worker): ok!
|
| 24 | 2026-08-25T21:03:56.621Z | INFO | crucible-agent (worker): SMF ok!
|
| 25 | 2026-08-25T21:03:58.610Z | INFO | crucible-agent (dropshot): request completed
latency_us = 63
local_addr = [fd00:1122:3344:101::14]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:40517
req_id = a861de1b-7ccf-4033-a467-cff0f55c63ec
response_code = 200
uri = /crucible/0/regions
|
| 26 | 2026-08-25T21:04:00.385Z | INFO | crucible-agent (dropshot): request completed
latency_us = 68
local_addr = [fd00:1122:3344:101::14]:32345
method = GET
remote_addr = [fd00:1122:3344:101::b]:40517
req_id = c9f7c51a-39e0-4835-a211-6e269da2b43c
response_code = 200
uri = /crucible/0/regions/729f52bc-7dbb-40c8-9fd4-59688c0736fd
|
| 27 | 2026-08-25T21:04:00.388Z | INFO | crucible-agent (datafile): path is "/data/regions/729f52bc-7dbb-40c8-9fd4-59688c0736fd"
|
| 28 | 2026-08-25T21:04:00.395Z | INFO | crucible-agent (datafile): dataset is oxp_a7733fa5-3cc8-42fc-829c-28163a3731c1/crucible/regions/729f52bc-7dbb-40c8-9fd4-59688c0736fd
|
| 29 | 2026-08-25T21:04:00.401Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_a7733fa5-3cc8-42fc-829c-28163a3731c1/crucible/regions/729f52bc-7dbb-40c8-9fd4-59688c0736fd@34cd36df-17fa-4604-865a-3fb946aee839
|
| 30 | 2026-08-25T21:04:00.401Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_a7733fa5-3cc8-42fc-829c-28163a3731c1/crucible/regions/729f52bc-7dbb-40c8-9fd4-59688c0736fd", "34cd36df-17fa-4604-865a-3fb946aee839"]
|
| 31 | 2026-08-25T21:04:00.412Z | INFO | crucible-agent (dropshot): request completed
latency_us = 23785
local_addr = [fd00:1122:3344:101::14]:32345
method = GET
remote_addr = [fd00:1122:3344:101::b]:40517
req_id = f6b06e75-d067-4421-b0f5-daabe1d19fc9
response_code = 200
uri = /crucible/0/regions/729f52bc-7dbb-40c8-9fd4-59688c0736fd/snapshots/34cd36df-17fa-4604-865a-3fb946aee839
|
| 32 | 2026-08-25T21:04:00.415Z | INFO | crucible-agent (datafile): path is "/data/regions/729f52bc-7dbb-40c8-9fd4-59688c0736fd"
|
| 33 | 2026-08-25T21:04:00.422Z | INFO | crucible-agent (datafile): dataset is oxp_a7733fa5-3cc8-42fc-829c-28163a3731c1/crucible/regions/729f52bc-7dbb-40c8-9fd4-59688c0736fd
|
| 34 | 2026-08-25T21:04:00.428Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_a7733fa5-3cc8-42fc-829c-28163a3731c1/crucible/regions/729f52bc-7dbb-40c8-9fd4-59688c0736fd@34cd36df-17fa-4604-865a-3fb946aee839
|
| 35 | 2026-08-25T21:04:00.428Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_a7733fa5-3cc8-42fc-829c-28163a3731c1/crucible/regions/729f52bc-7dbb-40c8-9fd4-59688c0736fd", "34cd36df-17fa-4604-865a-3fb946aee839"]
|
| 36 | 2026-08-25T21:04:00.436Z | INFO | crucible-agent (datafile): requesting running snapshot 729f52bc-7dbb-40c8-9fd4-59688c0736fd-34cd36df-17fa-4604-865a-3fb946aee839 state: Requested
|
| 37 | 2026-08-25T21:04:00.442Z | INFO | crucible-agent (worker): applying SMF actions for region 729f52bc-7dbb-40c8-9fd4-59688c0736fd running snapshot 34cd36df-17fa-4604-865a-3fb946aee839 (state Requested)...
|
| 38 | 2026-08-25T21:04:00.442Z | INFO | crucible-agent (dropshot): request completed
latency_us = 26864
local_addr = [fd00:1122:3344:101::14]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:40517
req_id = 7d949874-c3f9-4628-9585-f08369346640
response_code = 200
uri = /crucible/0/regions/729f52bc-7dbb-40c8-9fd4-59688c0736fd/snapshots/34cd36df-17fa-4604-865a-3fb946aee839/run
|
| 39 | 2026-08-25T21:04:00.444Z | INFO | crucible-agent (worker): creating missing snapshot instance snapshot-729f52bc-7dbb-40c8-9fd4-59688c0736fd-34cd36df-17fa-4604-865a-3fb946aee839
|
| 40 | 2026-08-25T21:04:00.480Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:snapshot-729f52bc-7dbb-40c8-9fd4-59688c0736fd-34cd36df-17fa-4604-865a-3fb946aee839
|
| 41 | 2026-08-25T21:04:00.515Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:snapshot-729f52bc-7dbb-40c8-9fd4-59688c0736fd-34cd36df-17fa-4604-865a-3fb946aee839
|
| 42 | 2026-08-25T21:04:00.516Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/729f52bc-7dbb-40c8-9fd4-59688c0736fd/.zfs/snapshot/34cd36df-17fa-4604-865a-3fb946aee839
|
| 43 | 2026-08-25T21:04:00.516Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19002
|
| 44 | 2026-08-25T21:04:00.516Z | INFO | crucible-agent (worker): ensure mode SCF_TYPE_ASTRING ro
|
| 45 | 2026-08-25T21:04:00.516Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14
|
| 46 | 2026-08-25T21:04:00.516Z | INFO | crucible-agent (worker): commit
|
| 47 | 2026-08-25T21:04:00.531Z | INFO | crucible-agent (worker): ok!
|
| 48 | 2026-08-25T21:04:00.544Z | INFO | crucible-agent (worker): SMF ok!
|
| 49 | 2026-08-25T21:04:00.544Z | INFO | crucible-agent (datafile): region 729f52bc-7dbb-40c8-9fd4-59688c0736fd running snapshot 34cd36df-17fa-4604-865a-3fb946aee839 state: Requested -> Created
|
| 50 | 2026-08-25T21:04:00.723Z | INFO | crucible-agent (datafile): path is "/data/regions/729f52bc-7dbb-40c8-9fd4-59688c0736fd"
|
| 51 | 2026-08-25T21:04:00.730Z | INFO | crucible-agent (datafile): dataset is oxp_a7733fa5-3cc8-42fc-829c-28163a3731c1/crucible/regions/729f52bc-7dbb-40c8-9fd4-59688c0736fd
|
| 52 | 2026-08-25T21:04:00.736Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_a7733fa5-3cc8-42fc-829c-28163a3731c1/crucible/regions/729f52bc-7dbb-40c8-9fd4-59688c0736fd@34cd36df-17fa-4604-865a-3fb946aee839
|
| 53 | 2026-08-25T21:04:00.736Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_a7733fa5-3cc8-42fc-829c-28163a3731c1/crucible/regions/729f52bc-7dbb-40c8-9fd4-59688c0736fd", "34cd36df-17fa-4604-865a-3fb946aee839"]
|
| 54 | 2026-08-25T21:04:00.741Z | INFO | crucible-agent (dropshot): request completed
latency_us = 17139
local_addr = [fd00:1122:3344:101::14]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:40517
req_id = 3daf5826-9322-4158-a18a-5abfb252a8fa
response_code = 200
uri = /crucible/0/regions/729f52bc-7dbb-40c8-9fd4-59688c0736fd/snapshots/34cd36df-17fa-4604-865a-3fb946aee839/run
|
| 55 | 2026-08-25T21:04:07.035Z | INFO | crucible-agent (datafile): region 88f77466-b33f-4f22-bd77-45f04436afd3 state: Requested
|
| 56 | 2026-08-25T21:04:07.035Z | INFO | crucible-agent (dropshot): request completed
latency_us = 225
local_addr = [fd00:1122:3344:101::14]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:40517
req_id = 07b88ad4-6520-4e55-9efd-b5dcaa87c717
response_code = 200
uri = /crucible/0/regions
|
| 57 | 2026-08-25T21:04:07.035Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 58 | 2026-08-25T21:04:07.042Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_a7733fa5-3cc8-42fc-829c-28163a3731c1/crucible/regions/88f77466-b33f-4f22-bd77-45f04436afd3
|
| 59 | 2026-08-25T21:04:07.042Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_a7733fa5-3cc8-42fc-829c-28163a3731c1/crucible/regions/88f77466-b33f-4f22-bd77-45f04436afd3
|
| 60 | 2026-08-25T21:04:07.147Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("88f77466-b33f-4f22-bd77-45f04436afd3"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/88f77466-b33f-4f22-bd77-45f04436afd3"
region = 88f77466-b33f-4f22-bd77-45f04436afd3
|
| 61 | 2026-08-25T21:04:07.147Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "88f77466-b33f-4f22-bd77-45f04436afd3" "--data" "/data/regions/88f77466-b33f-4f22-bd77-45f04436afd3" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = 88f77466-b33f-4f22-bd77-45f04436afd3
|
| 62 | 2026-08-25T21:04:07.393Z | INFO | crucible-agent (dropshot): request completed
latency_us = 70
local_addr = [fd00:1122:3344:101::14]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:40517
req_id = 560f4563-b819-44b8-9bdb-8d49a9e7ef2a
response_code = 200
uri = /crucible/0/regions
|
| 63 | 2026-08-25T21:04:07.437Z | INFO | crucible-agent (worker): region files created ok
region = 88f77466-b33f-4f22-bd77-45f04436afd3
|
| 64 | 2026-08-25T21:04:07.437Z | INFO | crucible-agent (datafile): region 88f77466-b33f-4f22-bd77-45f04436afd3 state: Requested -> Created
|
| 65 | 2026-08-25T21:04:07.438Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 66 | 2026-08-25T21:04:07.439Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-88f77466-b33f-4f22-bd77-45f04436afd3
|
| 67 | 2026-08-25T21:04:07.450Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-88f77466-b33f-4f22-bd77-45f04436afd3
|
| 68 | 2026-08-25T21:04:07.474Z | INFO | crucible-agent (worker): creating config property group
|
| 69 | 2026-08-25T21:04:07.486Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-88f77466-b33f-4f22-bd77-45f04436afd3
|
| 70 | 2026-08-25T21:04:07.486Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/88f77466-b33f-4f22-bd77-45f04436afd3
|
| 71 | 2026-08-25T21:04:07.486Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19003
|
| 72 | 2026-08-25T21:04:07.486Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14
|
| 73 | 2026-08-25T21:04:07.486Z | INFO | crucible-agent (worker): commit
|
| 74 | 2026-08-25T21:04:07.497Z | INFO | crucible-agent (worker): ok!
|
| 75 | 2026-08-25T21:04:07.510Z | INFO | crucible-agent (worker): SMF ok!
|
| 76 | 2026-08-25T21:04:07.790Z | INFO | crucible-agent (dropshot): request completed
latency_us = 56
local_addr = [fd00:1122:3344:101::14]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:40517
req_id = fe4fe3fa-7e23-44ca-8eb1-d98225bd1a09
response_code = 200
uri = /crucible/0/regions
|
| 77 | 2026-08-25T21:04:40.297Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::14]:32345
remote_addr = [fd00:1122:3344:101::b]:48390
|
| 78 | 2026-08-25T21:04:40.297Z | INFO | crucible-agent (dropshot): request completed
latency_us = 63
local_addr = [fd00:1122:3344:101::14]:32345
method = GET
remote_addr = [fd00:1122:3344:101::b]:48390
req_id = 850db8da-a411-4386-b1d0-81a77645e1f5
response_code = 200
uri = /crucible/0/regions/88f77466-b33f-4f22-bd77-45f04436afd3
|
| 79 | 2026-08-25T21:04:40.301Z | INFO | crucible-agent (datafile): path is "/data/regions/88f77466-b33f-4f22-bd77-45f04436afd3"
|
| 80 | 2026-08-25T21:04:40.313Z | INFO | crucible-agent (datafile): dataset is oxp_a7733fa5-3cc8-42fc-829c-28163a3731c1/crucible/regions/88f77466-b33f-4f22-bd77-45f04436afd3
|
| 81 | 2026-08-25T21:04:40.320Z | INFO | crucible-agent (datafile): region 88f77466-b33f-4f22-bd77-45f04436afd3 state: Created -> Tombstoned
|
| 82 | 2026-08-25T21:04:40.320Z | INFO | crucible-agent (worker): applying SMF actions before removal...
|
| 83 | 2026-08-25T21:04:40.320Z | INFO | crucible-agent (dropshot): request completed
latency_us = 19397
local_addr = [fd00:1122:3344:101::14]:32345
method = DELETE
remote_addr = [fd00:1122:3344:101::b]:48390
req_id = f1a78f89-fc34-4409-b078-9363608aab0e
response_code = 204
uri = /crucible/0/regions/88f77466-b33f-4f22-bd77-45f04436afd3
|
| 84 | 2026-08-25T21:04:40.321Z | INFO | crucible-agent (worker): disabling downstairs instance: downstairs-88f77466-b33f-4f22-bd77-45f04436afd3 (instance states: (Some(Online), None))
|
| 85 | 2026-08-25T21:04:40.324Z | INFO | crucible-agent (dropshot): request completed
latency_us = 39
local_addr = [fd00:1122:3344:101::14]:32345
method = GET
remote_addr = [fd00:1122:3344:101::b]:48390
req_id = 20366295-e6ff-47c9-96e3-4fd1655cc2e2
response_code = 200
uri = /crucible/0/regions/88f77466-b33f-4f22-bd77-45f04436afd3
|
| 86 | 2026-08-25T21:04:40.334Z | INFO | crucible-agent (worker): SMF ok!
|
| 87 | 2026-08-25T21:04:40.343Z | INFO | crucible-agent (worker): deleting zfs dataset "oxp_a7733fa5-3cc8-42fc-829c-28163a3731c1/crucible/regions/88f77466-b33f-4f22-bd77-45f04436afd3"
region = 88f77466-b33f-4f22-bd77-45f04436afd3
|
| 88 | 2026-08-25T21:04:40.402Z | INFO | crucible-agent (datafile): region 88f77466-b33f-4f22-bd77-45f04436afd3 state: Tombstoned -> Destroyed
|
| 89 | 2026-08-25T21:04:40.426Z | INFO | crucible-agent (dropshot): request completed
latency_us = 71
local_addr = [fd00:1122:3344:101::14]:32345
method = GET
remote_addr = [fd00:1122:3344:101::b]:48390
req_id = b219d519-7d34-4a73-9216-cf498ca4e7af
response_code = 200
uri = /crucible/0/regions/88f77466-b33f-4f22-bd77-45f04436afd3
|