| 1 | 2026-06-13T03:51:39.461Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::18]:32345
remote_addr = [fd00:1122:3344:101::c]:39371
|
| 2 | 2026-06-13T03:51:39.461Z | INFO | crucible-agent (datafile): region b8b0b181-c212-4227-a321-10c7eb1dfaaa state: Requested
|
| 3 | 2026-06-13T03:51:39.465Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 4 | 2026-06-13T03:51:39.465Z | INFO | crucible-agent (dropshot): request completed
latency_us = 4582
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::c]:39371
req_id = 711559e3-3429-4a98-ad7a-af8e86576db5
response_code = 200
uri = /crucible/0/regions
|
| 5 | 2026-06-13T03:51:39.473Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_ff19ca09-a993-4488-96dc-9cad19ddae8a/crucible/regions/b8b0b181-c212-4227-a321-10c7eb1dfaaa
|
| 6 | 2026-06-13T03:51:39.473Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_ff19ca09-a993-4488-96dc-9cad19ddae8a/crucible/regions/b8b0b181-c212-4227-a321-10c7eb1dfaaa
|
| 7 | 2026-06-13T03:51:39.523Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("b8b0b181-c212-4227-a321-10c7eb1dfaaa"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/b8b0b181-c212-4227-a321-10c7eb1dfaaa"
region = b8b0b181-c212-4227-a321-10c7eb1dfaaa
|
| 8 | 2026-06-13T03:51:39.523Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "b8b0b181-c212-4227-a321-10c7eb1dfaaa" "--data" "/data/regions/b8b0b181-c212-4227-a321-10c7eb1dfaaa" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = b8b0b181-c212-4227-a321-10c7eb1dfaaa
|
| 9 | 2026-06-13T03:51:39.573Z | INFO | crucible-agent (worker): region files created ok
region = b8b0b181-c212-4227-a321-10c7eb1dfaaa
|
| 10 | 2026-06-13T03:51:39.573Z | INFO | crucible-agent (datafile): region b8b0b181-c212-4227-a321-10c7eb1dfaaa state: Requested -> Created
|
| 11 | 2026-06-13T03:51:39.573Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 12 | 2026-06-13T03:51:39.589Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-b8b0b181-c212-4227-a321-10c7eb1dfaaa
|
| 13 | 2026-06-13T03:51:39.592Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-b8b0b181-c212-4227-a321-10c7eb1dfaaa
|
| 14 | 2026-06-13T03:51:39.598Z | INFO | crucible-agent (worker): creating config property group
|
| 15 | 2026-06-13T03:51:39.599Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-b8b0b181-c212-4227-a321-10c7eb1dfaaa
|
| 16 | 2026-06-13T03:51:39.600Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/b8b0b181-c212-4227-a321-10c7eb1dfaaa
|
| 17 | 2026-06-13T03:51:39.600Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19001
|
| 18 | 2026-06-13T03:51:39.600Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18
|
| 19 | 2026-06-13T03:51:39.600Z | INFO | crucible-agent (worker): commit
|
| 20 | 2026-06-13T03:51:39.601Z | INFO | crucible-agent (worker): ok!
|
| 21 | 2026-06-13T03:51:39.604Z | INFO | crucible-agent (worker): SMF ok!
|
| 22 | 2026-06-13T03:51:39.682Z | INFO | crucible-agent (dropshot): request completed
latency_us = 59
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::c]:39371
req_id = 97a8f1af-1a54-4dcf-a1ca-ded545ebf257
response_code = 200
uri = /crucible/0/regions
|
| 23 | 2026-06-13T03:51:40.979Z | INFO | crucible-agent (dropshot): request completed
latency_us = 74
local_addr = [fd00:1122:3344:101::18]:32345
method = GET
remote_addr = [fd00:1122:3344:101::c]:39371
req_id = c604dcdc-2ee2-43d8-94d4-88a80a68b6e5
response_code = 200
uri = /crucible/0/regions/96793e7f-8f0d-42a6-9938-dd75f8758e92
|
| 24 | 2026-06-13T03:51:40.982Z | INFO | crucible-agent (datafile): path is "/data/regions/96793e7f-8f0d-42a6-9938-dd75f8758e92"
|
| 25 | 2026-06-13T03:51:40.989Z | INFO | crucible-agent (datafile): dataset is oxp_ff19ca09-a993-4488-96dc-9cad19ddae8a/crucible/regions/96793e7f-8f0d-42a6-9938-dd75f8758e92
|
| 26 | 2026-06-13T03:51:40.995Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_ff19ca09-a993-4488-96dc-9cad19ddae8a/crucible/regions/96793e7f-8f0d-42a6-9938-dd75f8758e92@7b06fb80-18ec-485b-8d74-dc37273f46c2
|
| 27 | 2026-06-13T03:51:40.995Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_ff19ca09-a993-4488-96dc-9cad19ddae8a/crucible/regions/96793e7f-8f0d-42a6-9938-dd75f8758e92", "7b06fb80-18ec-485b-8d74-dc37273f46c2"]
|
| 28 | 2026-06-13T03:51:41.003Z | INFO | crucible-agent (dropshot): request completed
latency_us = 20912
local_addr = [fd00:1122:3344:101::18]:32345
method = GET
remote_addr = [fd00:1122:3344:101::c]:39371
req_id = 48861211-6806-49ab-a16e-d15c15c9520d
response_code = 200
uri = /crucible/0/regions/96793e7f-8f0d-42a6-9938-dd75f8758e92/snapshots/7b06fb80-18ec-485b-8d74-dc37273f46c2
|
| 29 | 2026-06-13T03:51:41.006Z | INFO | crucible-agent (datafile): path is "/data/regions/96793e7f-8f0d-42a6-9938-dd75f8758e92"
|
| 30 | 2026-06-13T03:51:41.013Z | INFO | crucible-agent (datafile): dataset is oxp_ff19ca09-a993-4488-96dc-9cad19ddae8a/crucible/regions/96793e7f-8f0d-42a6-9938-dd75f8758e92
|
| 31 | 2026-06-13T03:51:41.019Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_ff19ca09-a993-4488-96dc-9cad19ddae8a/crucible/regions/96793e7f-8f0d-42a6-9938-dd75f8758e92@7b06fb80-18ec-485b-8d74-dc37273f46c2
|
| 32 | 2026-06-13T03:51:41.019Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_ff19ca09-a993-4488-96dc-9cad19ddae8a/crucible/regions/96793e7f-8f0d-42a6-9938-dd75f8758e92", "7b06fb80-18ec-485b-8d74-dc37273f46c2"]
|
| 33 | 2026-06-13T03:51:41.027Z | INFO | crucible-agent (datafile): requesting running snapshot 96793e7f-8f0d-42a6-9938-dd75f8758e92-7b06fb80-18ec-485b-8d74-dc37273f46c2 state: Requested
|
| 34 | 2026-06-13T03:51:41.027Z | INFO | crucible-agent (worker): applying SMF actions for region 96793e7f-8f0d-42a6-9938-dd75f8758e92 running snapshot 7b06fb80-18ec-485b-8d74-dc37273f46c2 (state Requested)...
|
| 35 | 2026-06-13T03:51:41.027Z | INFO | crucible-agent (dropshot): request completed
latency_us = 20655
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::c]:39371
req_id = 1257f430-eee8-4d61-8f89-8f13a2b3afe8
response_code = 200
uri = /crucible/0/regions/96793e7f-8f0d-42a6-9938-dd75f8758e92/snapshots/7b06fb80-18ec-485b-8d74-dc37273f46c2/run
|
| 36 | 2026-06-13T03:51:41.032Z | INFO | crucible-agent (worker): creating missing snapshot instance snapshot-96793e7f-8f0d-42a6-9938-dd75f8758e92-7b06fb80-18ec-485b-8d74-dc37273f46c2
|
| 37 | 2026-06-13T03:51:41.041Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:snapshot-96793e7f-8f0d-42a6-9938-dd75f8758e92-7b06fb80-18ec-485b-8d74-dc37273f46c2
|
| 38 | 2026-06-13T03:51:41.046Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:snapshot-96793e7f-8f0d-42a6-9938-dd75f8758e92-7b06fb80-18ec-485b-8d74-dc37273f46c2
|
| 39 | 2026-06-13T03:51:41.046Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/96793e7f-8f0d-42a6-9938-dd75f8758e92/.zfs/snapshot/7b06fb80-18ec-485b-8d74-dc37273f46c2
|
| 40 | 2026-06-13T03:51:41.046Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19002
|
| 41 | 2026-06-13T03:51:41.047Z | INFO | crucible-agent (worker): ensure mode SCF_TYPE_ASTRING ro
|
| 42 | 2026-06-13T03:51:41.047Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18
|
| 43 | 2026-06-13T03:51:41.047Z | INFO | crucible-agent (worker): commit
|
| 44 | 2026-06-13T03:51:41.049Z | INFO | crucible-agent (worker): ok!
|
| 45 | 2026-06-13T03:51:41.052Z | INFO | crucible-agent (worker): SMF ok!
|
| 46 | 2026-06-13T03:51:41.052Z | INFO | crucible-agent (datafile): region 96793e7f-8f0d-42a6-9938-dd75f8758e92 running snapshot 7b06fb80-18ec-485b-8d74-dc37273f46c2 state: Requested -> Created
|
| 47 | 2026-06-13T03:51:41.360Z | INFO | crucible-agent (datafile): path is "/data/regions/96793e7f-8f0d-42a6-9938-dd75f8758e92"
|
| 48 | 2026-06-13T03:51:41.367Z | INFO | crucible-agent (datafile): dataset is oxp_ff19ca09-a993-4488-96dc-9cad19ddae8a/crucible/regions/96793e7f-8f0d-42a6-9938-dd75f8758e92
|
| 49 | 2026-06-13T03:51:41.374Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_ff19ca09-a993-4488-96dc-9cad19ddae8a/crucible/regions/96793e7f-8f0d-42a6-9938-dd75f8758e92@7b06fb80-18ec-485b-8d74-dc37273f46c2
|
| 50 | 2026-06-13T03:51:41.374Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_ff19ca09-a993-4488-96dc-9cad19ddae8a/crucible/regions/96793e7f-8f0d-42a6-9938-dd75f8758e92", "7b06fb80-18ec-485b-8d74-dc37273f46c2"]
|
| 51 | 2026-06-13T03:51:41.379Z | INFO | crucible-agent (dropshot): request completed
latency_us = 19379
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::c]:39371
req_id = fce4bf44-e39d-4fc3-bc01-f711df66bdfa
response_code = 200
uri = /crucible/0/regions/96793e7f-8f0d-42a6-9938-dd75f8758e92/snapshots/7b06fb80-18ec-485b-8d74-dc37273f46c2/run
|
| 52 | 2026-06-13T03:51:46.083Z | INFO | crucible-agent (datafile): region fd9bc778-7929-4b79-891d-09440a786e6c state: Requested
|
| 53 | 2026-06-13T03:51:46.084Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 54 | 2026-06-13T03:51:46.084Z | INFO | crucible-agent (dropshot): request completed
latency_us = 236
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::c]:39371
req_id = 0adcba36-748f-4078-99d2-673b5d649c15
response_code = 200
uri = /crucible/0/regions
|
| 55 | 2026-06-13T03:51:46.089Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_ff19ca09-a993-4488-96dc-9cad19ddae8a/crucible/regions/fd9bc778-7929-4b79-891d-09440a786e6c
|
| 56 | 2026-06-13T03:51:46.089Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_ff19ca09-a993-4488-96dc-9cad19ddae8a/crucible/regions/fd9bc778-7929-4b79-891d-09440a786e6c
|
| 57 | 2026-06-13T03:51:46.127Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("fd9bc778-7929-4b79-891d-09440a786e6c"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/fd9bc778-7929-4b79-891d-09440a786e6c"
region = fd9bc778-7929-4b79-891d-09440a786e6c
|
| 58 | 2026-06-13T03:51:46.128Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "fd9bc778-7929-4b79-891d-09440a786e6c" "--data" "/data/regions/fd9bc778-7929-4b79-891d-09440a786e6c" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = fd9bc778-7929-4b79-891d-09440a786e6c
|
| 59 | 2026-06-13T03:51:46.174Z | INFO | crucible-agent (worker): region files created ok
region = fd9bc778-7929-4b79-891d-09440a786e6c
|
| 60 | 2026-06-13T03:51:46.174Z | INFO | crucible-agent (datafile): region fd9bc778-7929-4b79-891d-09440a786e6c state: Requested -> Created
|
| 61 | 2026-06-13T03:51:46.174Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 62 | 2026-06-13T03:51:46.176Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-fd9bc778-7929-4b79-891d-09440a786e6c
|
| 63 | 2026-06-13T03:51:46.177Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-fd9bc778-7929-4b79-891d-09440a786e6c
|
| 64 | 2026-06-13T03:51:46.181Z | INFO | crucible-agent (worker): creating config property group
|
| 65 | 2026-06-13T03:51:46.182Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-fd9bc778-7929-4b79-891d-09440a786e6c
|
| 66 | 2026-06-13T03:51:46.182Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/fd9bc778-7929-4b79-891d-09440a786e6c
|
| 67 | 2026-06-13T03:51:46.182Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19003
|
| 68 | 2026-06-13T03:51:46.183Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18
|
| 69 | 2026-06-13T03:51:46.183Z | INFO | crucible-agent (worker): commit
|
| 70 | 2026-06-13T03:51:46.184Z | INFO | crucible-agent (worker): ok!
|
| 71 | 2026-06-13T03:51:46.188Z | INFO | crucible-agent (worker): SMF ok!
|
| 72 | 2026-06-13T03:51:46.376Z | INFO | crucible-agent (dropshot): request completed
latency_us = 67
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::c]:39371
req_id = 349ee9ba-b736-4480-9f4e-cb9ee44b950a
response_code = 200
uri = /crucible/0/regions
|
| 73 | 2026-06-13T03:52:11.144Z | INFO | crucible-agent (dropshot): request completed
latency_us = 53
local_addr = [fd00:1122:3344:101::18]:32345
method = GET
remote_addr = [fd00:1122:3344:101::c]:39371
req_id = 99c60051-f455-4303-aed0-733b988fc13c
response_code = 200
uri = /crucible/0/regions/fd9bc778-7929-4b79-891d-09440a786e6c
|
| 74 | 2026-06-13T03:52:11.148Z | INFO | crucible-agent (datafile): path is "/data/regions/fd9bc778-7929-4b79-891d-09440a786e6c"
|
| 75 | 2026-06-13T03:52:11.161Z | INFO | crucible-agent (datafile): dataset is oxp_ff19ca09-a993-4488-96dc-9cad19ddae8a/crucible/regions/fd9bc778-7929-4b79-891d-09440a786e6c
|
| 76 | 2026-06-13T03:52:11.169Z | INFO | crucible-agent (datafile): region fd9bc778-7929-4b79-891d-09440a786e6c state: Created -> Tombstoned
|
| 77 | 2026-06-13T03:52:11.170Z | INFO | crucible-agent (worker): applying SMF actions before removal...
|
| 78 | 2026-06-13T03:52:11.170Z | INFO | crucible-agent (dropshot): request completed
latency_us = 21574
local_addr = [fd00:1122:3344:101::18]:32345
method = DELETE
remote_addr = [fd00:1122:3344:101::c]:39371
req_id = 5ef1b61f-e284-497c-af63-888bd45dd647
response_code = 204
uri = /crucible/0/regions/fd9bc778-7929-4b79-891d-09440a786e6c
|
| 79 | 2026-06-13T03:52:11.171Z | INFO | crucible-agent (worker): disabling downstairs instance: downstairs-fd9bc778-7929-4b79-891d-09440a786e6c (instance states: (Some(Online), None))
|
| 80 | 2026-06-13T03:52:11.174Z | INFO | crucible-agent (dropshot): request completed
latency_us = 51
local_addr = [fd00:1122:3344:101::18]:32345
method = GET
remote_addr = [fd00:1122:3344:101::c]:39371
req_id = 886db696-7b57-4a85-8611-e5f5fb6408f6
response_code = 200
uri = /crucible/0/regions/fd9bc778-7929-4b79-891d-09440a786e6c
|
| 81 | 2026-06-13T03:52:11.178Z | INFO | crucible-agent (worker): SMF ok!
|
| 82 | 2026-06-13T03:52:11.187Z | INFO | crucible-agent (worker): deleting zfs dataset "oxp_ff19ca09-a993-4488-96dc-9cad19ddae8a/crucible/regions/fd9bc778-7929-4b79-891d-09440a786e6c"
region = fd9bc778-7929-4b79-891d-09440a786e6c
|
| 83 | 2026-06-13T03:52:11.233Z | INFO | crucible-agent (datafile): region fd9bc778-7929-4b79-891d-09440a786e6c state: Tombstoned -> Destroyed
|
| 84 | 2026-06-13T03:52:11.262Z | INFO | crucible-agent (dropshot): request completed
latency_us = 74
local_addr = [fd00:1122:3344:101::18]:32345
method = GET
remote_addr = [fd00:1122:3344:101::c]:39371
req_id = 37d8df3c-9932-4eb8-b076-974d2c57eca5
response_code = 200
uri = /crucible/0/regions/fd9bc778-7929-4b79-891d-09440a786e6c
|