| 1 | 2026-05-19T20:15:08.937Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::13]:32345
remote_addr = [fd00:1122:3344:101::a]:35803
|
| 2 | 2026-05-19T20:15:08.937Z | INFO | crucible-agent (datafile): region 8d37d639-8f03-406b-b0ad-13fc7a71bf15 state: Requested
|
| 3 | 2026-05-19T20:15:08.939Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 4 | 2026-05-19T20:15:08.939Z | INFO | crucible-agent (dropshot): request completed
latency_us = 2270
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:35803
req_id = e7046793-887c-4974-95c1-a78a51971891
response_code = 200
uri = /crucible/0/regions
|
| 5 | 2026-05-19T20:15:08.947Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_6d978576-2d94-4186-b878-41638679f7d2/crucible/regions/8d37d639-8f03-406b-b0ad-13fc7a71bf15
|
| 6 | 2026-05-19T20:15:08.947Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_6d978576-2d94-4186-b878-41638679f7d2/crucible/regions/8d37d639-8f03-406b-b0ad-13fc7a71bf15
|
| 7 | 2026-05-19T20:15:09.102Z | INFO | crucible-agent (dropshot): request completed
latency_us = 51
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:35803
req_id = fcef7a5d-1535-4f2e-923f-dc8668ac3ba9
response_code = 200
uri = /crucible/0/regions
|
| 8 | 2026-05-19T20:15:09.672Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("8d37d639-8f03-406b-b0ad-13fc7a71bf15"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/8d37d639-8f03-406b-b0ad-13fc7a71bf15"
region = 8d37d639-8f03-406b-b0ad-13fc7a71bf15
|
| 9 | 2026-05-19T20:15:09.672Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "8d37d639-8f03-406b-b0ad-13fc7a71bf15" "--data" "/data/regions/8d37d639-8f03-406b-b0ad-13fc7a71bf15" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = 8d37d639-8f03-406b-b0ad-13fc7a71bf15
|
| 10 | 2026-05-19T20:15:09.674Z | INFO | crucible-agent (dropshot): request completed
latency_us = 42
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:35803
req_id = df586b38-6b62-41cf-8aaf-51b917c68a43
response_code = 200
uri = /crucible/0/regions
|
| 11 | 2026-05-19T20:15:09.774Z | INFO | crucible-agent (worker): region files created ok
region = 8d37d639-8f03-406b-b0ad-13fc7a71bf15
|
| 12 | 2026-05-19T20:15:09.774Z | INFO | crucible-agent (datafile): region 8d37d639-8f03-406b-b0ad-13fc7a71bf15 state: Requested -> Created
|
| 13 | 2026-05-19T20:15:09.774Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 14 | 2026-05-19T20:15:09.774Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-8d37d639-8f03-406b-b0ad-13fc7a71bf15
|
| 15 | 2026-05-19T20:15:09.785Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-8d37d639-8f03-406b-b0ad-13fc7a71bf15
|
| 16 | 2026-05-19T20:15:09.809Z | INFO | crucible-agent (worker): creating config property group
|
| 17 | 2026-05-19T20:15:09.812Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-8d37d639-8f03-406b-b0ad-13fc7a71bf15
|
| 18 | 2026-05-19T20:15:09.812Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/8d37d639-8f03-406b-b0ad-13fc7a71bf15
|
| 19 | 2026-05-19T20:15:09.812Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19001
|
| 20 | 2026-05-19T20:15:09.812Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13
|
| 21 | 2026-05-19T20:15:09.812Z | INFO | crucible-agent (worker): commit
|
| 22 | 2026-05-19T20:15:09.816Z | INFO | crucible-agent (worker): ok!
|
| 23 | 2026-05-19T20:15:09.833Z | INFO | crucible-agent (worker): SMF ok!
|
| 24 | 2026-05-19T20:15:10.384Z | INFO | crucible-agent (dropshot): request completed
latency_us = 70
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:35803
req_id = ec3ffd5f-9807-47cf-a260-734533e141b8
response_code = 200
uri = /crucible/0/regions
|
| 25 | 2026-05-19T20:15:11.663Z | INFO | crucible-agent (dropshot): request completed
latency_us = 67
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:35803
req_id = 4b3e5fdf-1683-457d-a2c1-98a3d748a84f
response_code = 200
uri = /crucible/0/regions/ba076c8b-4ace-4b91-b2b4-153870d226f9
|
| 26 | 2026-05-19T20:15:11.666Z | INFO | crucible-agent (datafile): path is "/data/regions/ba076c8b-4ace-4b91-b2b4-153870d226f9"
|
| 27 | 2026-05-19T20:15:11.672Z | INFO | crucible-agent (datafile): dataset is oxp_6d978576-2d94-4186-b878-41638679f7d2/crucible/regions/ba076c8b-4ace-4b91-b2b4-153870d226f9
|
| 28 | 2026-05-19T20:15:11.679Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_6d978576-2d94-4186-b878-41638679f7d2/crucible/regions/ba076c8b-4ace-4b91-b2b4-153870d226f9@67819a7a-9ac6-41e3-9914-01e877dd2567
|
| 29 | 2026-05-19T20:15:11.679Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_6d978576-2d94-4186-b878-41638679f7d2/crucible/regions/ba076c8b-4ace-4b91-b2b4-153870d226f9", "67819a7a-9ac6-41e3-9914-01e877dd2567"]
|
| 30 | 2026-05-19T20:15:11.687Z | INFO | crucible-agent (dropshot): request completed
latency_us = 20671
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:35803
req_id = 96d883d2-9778-4687-8c60-ebf6cdb935e5
response_code = 200
uri = /crucible/0/regions/ba076c8b-4ace-4b91-b2b4-153870d226f9/snapshots/67819a7a-9ac6-41e3-9914-01e877dd2567
|
| 31 | 2026-05-19T20:15:11.691Z | INFO | crucible-agent (datafile): path is "/data/regions/ba076c8b-4ace-4b91-b2b4-153870d226f9"
|
| 32 | 2026-05-19T20:15:11.697Z | INFO | crucible-agent (datafile): dataset is oxp_6d978576-2d94-4186-b878-41638679f7d2/crucible/regions/ba076c8b-4ace-4b91-b2b4-153870d226f9
|
| 33 | 2026-05-19T20:15:11.703Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_6d978576-2d94-4186-b878-41638679f7d2/crucible/regions/ba076c8b-4ace-4b91-b2b4-153870d226f9@67819a7a-9ac6-41e3-9914-01e877dd2567
|
| 34 | 2026-05-19T20:15:11.703Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_6d978576-2d94-4186-b878-41638679f7d2/crucible/regions/ba076c8b-4ace-4b91-b2b4-153870d226f9", "67819a7a-9ac6-41e3-9914-01e877dd2567"]
|
| 35 | 2026-05-19T20:15:11.711Z | INFO | crucible-agent (datafile): requesting running snapshot ba076c8b-4ace-4b91-b2b4-153870d226f9-67819a7a-9ac6-41e3-9914-01e877dd2567 state: Requested
|
| 36 | 2026-05-19T20:15:11.711Z | INFO | crucible-agent (worker): applying SMF actions for region ba076c8b-4ace-4b91-b2b4-153870d226f9 running snapshot 67819a7a-9ac6-41e3-9914-01e877dd2567 (state Requested)...
|
| 37 | 2026-05-19T20:15:11.711Z | INFO | crucible-agent (dropshot): request completed
latency_us = 19889
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:35803
req_id = 034ddc9d-59ce-4d2c-a00d-753692a33479
response_code = 200
uri = /crucible/0/regions/ba076c8b-4ace-4b91-b2b4-153870d226f9/snapshots/67819a7a-9ac6-41e3-9914-01e877dd2567/run
|
| 38 | 2026-05-19T20:15:11.716Z | INFO | crucible-agent (worker): creating missing snapshot instance snapshot-ba076c8b-4ace-4b91-b2b4-153870d226f9-67819a7a-9ac6-41e3-9914-01e877dd2567
|
| 39 | 2026-05-19T20:15:11.725Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:snapshot-ba076c8b-4ace-4b91-b2b4-153870d226f9-67819a7a-9ac6-41e3-9914-01e877dd2567
|
| 40 | 2026-05-19T20:15:11.735Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:snapshot-ba076c8b-4ace-4b91-b2b4-153870d226f9-67819a7a-9ac6-41e3-9914-01e877dd2567
|
| 41 | 2026-05-19T20:15:11.736Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/ba076c8b-4ace-4b91-b2b4-153870d226f9/.zfs/snapshot/67819a7a-9ac6-41e3-9914-01e877dd2567
|
| 42 | 2026-05-19T20:15:11.736Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19002
|
| 43 | 2026-05-19T20:15:11.736Z | INFO | crucible-agent (worker): ensure mode SCF_TYPE_ASTRING ro
|
| 44 | 2026-05-19T20:15:11.736Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13
|
| 45 | 2026-05-19T20:15:11.736Z | INFO | crucible-agent (worker): commit
|
| 46 | 2026-05-19T20:15:11.739Z | INFO | crucible-agent (worker): ok!
|
| 47 | 2026-05-19T20:15:11.744Z | INFO | crucible-agent (worker): SMF ok!
|
| 48 | 2026-05-19T20:15:11.744Z | INFO | crucible-agent (datafile): region ba076c8b-4ace-4b91-b2b4-153870d226f9 running snapshot 67819a7a-9ac6-41e3-9914-01e877dd2567 state: Requested -> Created
|
| 49 | 2026-05-19T20:15:12.012Z | INFO | crucible-agent (datafile): path is "/data/regions/ba076c8b-4ace-4b91-b2b4-153870d226f9"
|
| 50 | 2026-05-19T20:15:12.018Z | INFO | crucible-agent (datafile): dataset is oxp_6d978576-2d94-4186-b878-41638679f7d2/crucible/regions/ba076c8b-4ace-4b91-b2b4-153870d226f9
|
| 51 | 2026-05-19T20:15:12.025Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_6d978576-2d94-4186-b878-41638679f7d2/crucible/regions/ba076c8b-4ace-4b91-b2b4-153870d226f9@67819a7a-9ac6-41e3-9914-01e877dd2567
|
| 52 | 2026-05-19T20:15:12.025Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_6d978576-2d94-4186-b878-41638679f7d2/crucible/regions/ba076c8b-4ace-4b91-b2b4-153870d226f9", "67819a7a-9ac6-41e3-9914-01e877dd2567"]
|
| 53 | 2026-05-19T20:15:12.031Z | INFO | crucible-agent (dropshot): request completed
latency_us = 19180
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:35803
req_id = d379992c-0cee-47e9-9cff-d44ea4ddcb7b
response_code = 200
uri = /crucible/0/regions/ba076c8b-4ace-4b91-b2b4-153870d226f9/snapshots/67819a7a-9ac6-41e3-9914-01e877dd2567/run
|
| 54 | 2026-05-19T20:15:16.420Z | INFO | crucible-agent (datafile): region 1ac00c0e-c7e1-49e0-b545-f03e9c94ee20 state: Requested
|
| 55 | 2026-05-19T20:15:16.420Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 56 | 2026-05-19T20:15:16.420Z | INFO | crucible-agent (dropshot): request completed
latency_us = 269
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:35803
req_id = c205809c-6066-4733-9834-3c10152f1dab
response_code = 200
uri = /crucible/0/regions
|
| 57 | 2026-05-19T20:15:16.426Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_6d978576-2d94-4186-b878-41638679f7d2/crucible/regions/1ac00c0e-c7e1-49e0-b545-f03e9c94ee20
|
| 58 | 2026-05-19T20:15:16.426Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_6d978576-2d94-4186-b878-41638679f7d2/crucible/regions/1ac00c0e-c7e1-49e0-b545-f03e9c94ee20
|
| 59 | 2026-05-19T20:15:16.473Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("1ac00c0e-c7e1-49e0-b545-f03e9c94ee20"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/1ac00c0e-c7e1-49e0-b545-f03e9c94ee20"
region = 1ac00c0e-c7e1-49e0-b545-f03e9c94ee20
|
| 60 | 2026-05-19T20:15:16.473Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "1ac00c0e-c7e1-49e0-b545-f03e9c94ee20" "--data" "/data/regions/1ac00c0e-c7e1-49e0-b545-f03e9c94ee20" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = 1ac00c0e-c7e1-49e0-b545-f03e9c94ee20
|
| 61 | 2026-05-19T20:15:16.550Z | INFO | crucible-agent (worker): region files created ok
region = 1ac00c0e-c7e1-49e0-b545-f03e9c94ee20
|
| 62 | 2026-05-19T20:15:16.551Z | INFO | crucible-agent (datafile): region 1ac00c0e-c7e1-49e0-b545-f03e9c94ee20 state: Requested -> Created
|
| 63 | 2026-05-19T20:15:16.551Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 64 | 2026-05-19T20:15:16.551Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-1ac00c0e-c7e1-49e0-b545-f03e9c94ee20
|
| 65 | 2026-05-19T20:15:16.554Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-1ac00c0e-c7e1-49e0-b545-f03e9c94ee20
|
| 66 | 2026-05-19T20:15:16.561Z | INFO | crucible-agent (worker): creating config property group
|
| 67 | 2026-05-19T20:15:16.564Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-1ac00c0e-c7e1-49e0-b545-f03e9c94ee20
|
| 68 | 2026-05-19T20:15:16.564Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/1ac00c0e-c7e1-49e0-b545-f03e9c94ee20
|
| 69 | 2026-05-19T20:15:16.564Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19003
|
| 70 | 2026-05-19T20:15:16.564Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13
|
| 71 | 2026-05-19T20:15:16.564Z | INFO | crucible-agent (worker): commit
|
| 72 | 2026-05-19T20:15:16.568Z | INFO | crucible-agent (worker): ok!
|
| 73 | 2026-05-19T20:15:16.578Z | INFO | crucible-agent (worker): SMF ok!
|
| 74 | 2026-05-19T20:15:16.743Z | INFO | crucible-agent (dropshot): request completed
latency_us = 71
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:35803
req_id = b2e4c9b6-ff32-4574-90de-be791c73b9ca
response_code = 200
uri = /crucible/0/regions
|
| 75 | 2026-05-19T20:15:43.886Z | INFO | crucible-agent (dropshot): request completed
latency_us = 53
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:35803
req_id = 31bcf596-7d58-4a39-b193-69c528fd48b9
response_code = 200
uri = /crucible/0/regions/1ac00c0e-c7e1-49e0-b545-f03e9c94ee20
|
| 76 | 2026-05-19T20:15:43.889Z | INFO | crucible-agent (datafile): path is "/data/regions/1ac00c0e-c7e1-49e0-b545-f03e9c94ee20"
|
| 77 | 2026-05-19T20:15:43.901Z | INFO | crucible-agent (datafile): dataset is oxp_6d978576-2d94-4186-b878-41638679f7d2/crucible/regions/1ac00c0e-c7e1-49e0-b545-f03e9c94ee20
|
| 78 | 2026-05-19T20:15:43.908Z | INFO | crucible-agent (datafile): region 1ac00c0e-c7e1-49e0-b545-f03e9c94ee20 state: Created -> Tombstoned
|
| 79 | 2026-05-19T20:15:43.909Z | INFO | crucible-agent (worker): applying SMF actions before removal...
|
| 80 | 2026-05-19T20:15:43.909Z | INFO | crucible-agent (dropshot): request completed
latency_us = 19446
local_addr = [fd00:1122:3344:101::13]:32345
method = DELETE
remote_addr = [fd00:1122:3344:101::a]:35803
req_id = ab3db564-ae84-4b45-8930-4e4d6da04d33
response_code = 204
uri = /crucible/0/regions/1ac00c0e-c7e1-49e0-b545-f03e9c94ee20
|
| 81 | 2026-05-19T20:15:43.911Z | INFO | crucible-agent (worker): disabling downstairs instance: downstairs-1ac00c0e-c7e1-49e0-b545-f03e9c94ee20 (instance states: (Some(Online), None))
|
| 82 | 2026-05-19T20:15:43.911Z | INFO | crucible-agent (dropshot): request completed
latency_us = 30
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:35803
req_id = b3138d61-d000-40bd-b3b8-8083f899b607
response_code = 200
uri = /crucible/0/regions/1ac00c0e-c7e1-49e0-b545-f03e9c94ee20
|
| 83 | 2026-05-19T20:15:43.927Z | INFO | crucible-agent (worker): SMF ok!
|
| 84 | 2026-05-19T20:15:43.935Z | INFO | crucible-agent (worker): deleting zfs dataset "oxp_6d978576-2d94-4186-b878-41638679f7d2/crucible/regions/1ac00c0e-c7e1-49e0-b545-f03e9c94ee20"
region = 1ac00c0e-c7e1-49e0-b545-f03e9c94ee20
|
| 85 | 2026-05-19T20:15:43.993Z | INFO | crucible-agent (datafile): region 1ac00c0e-c7e1-49e0-b545-f03e9c94ee20 state: Tombstoned -> Destroyed
|
| 86 | 2026-05-19T20:15:44.032Z | INFO | crucible-agent (dropshot): request completed
latency_us = 45
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:35803
req_id = 0383220d-060c-4fbd-a19a-2e46c8d2f623
response_code = 200
uri = /crucible/0/regions/1ac00c0e-c7e1-49e0-b545-f03e9c94ee20
|