| 1 | | [ Sep 9 23:01:16 Enabled. ] |
| 2 | | [ Sep 9 23:01:16 Rereading configuration. ] |
| 3 | | [ Sep 9 23:01:18 Rereading configuration. ] |
| 4 | | [ Sep 9 23:01:22 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_a5e1ea0b-7e8d-4c32-affe-b8fb952236ec/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ] |
| 5 | | note: configured to log to "/dev/stdout" |
| 6 | 2025-09-09T23:01:22.904Z | INFO | crucible-agent: Crucible Version: Crucible Version: 0.0.1
Commit SHA: 779775d5130ff7a4836f52f48b7e64d1479ee104
Commit timestamp: 2025-08-09T00:27:59.000000000Z branch: main
rustc: 1.86.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos Debug: false Opt level: 3
|
| 7 | 2025-09-09T23:01:22.907Z | INFO | crucible-agent: dataset: "oxp_a5e1ea0b-7e8d-4c32-affe-b8fb952236ec/crucible"
|
| 8 | 2025-09-09T23:01:22.907Z | INFO | crucible-agent: listen IP: [fd00:1122:3344:101::14]:32345
|
| 9 | 2025-09-09T23:01:22.907Z | INFO | crucible-agent: SMF instance name downstairs_prefix: "downstairs"
|
| 10 | 2025-09-09T23:01:22.923Z | INFO | crucible-agent (datafile): Using conf_path:"/data/crucible.json"
|
| 11 | 2025-09-09T23:01:23.011Z | INFO | crucible-agent (dropshot): listening
local_addr = [fd00:1122:3344:101::14]:32345
|
| 12 | 2025-09-09T23:02:09.613Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::14]:32345
remote_addr = [fd00:1122:3344:101::b]:45621
|
| 13 | 2025-09-09T23:02:09.647Z | INFO | crucible-agent (datafile): region 3ced66f6-8325-4509-802a-c6ffede8666b state: Requested
|
| 14 | 2025-09-09T23:02:09.652Z | INFO | crucible-agent (dropshot): request completed
latency_us = 17179
local_addr = [fd00:1122:3344:101::14]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:45621
req_id = 83b4301c-1168-4880-bd66-eec83efb1ce8
response_code = 200
uri = /crucible/0/regions
|
| 15 | 2025-09-09T23:02:09.655Z | INFO | crucible-agent (worker): Region size:1073741824 reservation:1342177280 quota:3221225472
|
| 16 | 2025-09-09T23:02:09.664Z | INFO | crucible-agent (worker): zfs set reservation of 1342177280 for oxp_a5e1ea0b-7e8d-4c32-affe-b8fb952236ec/crucible/regions/3ced66f6-8325-4509-802a-c6ffede8666b
|
| 17 | 2025-09-09T23:02:09.664Z | INFO | crucible-agent (worker): zfs set quota of 3221225472 for oxp_a5e1ea0b-7e8d-4c32-affe-b8fb952236ec/crucible/regions/3ced66f6-8325-4509-802a-c6ffede8666b
|
| 18 | 2025-09-09T23:02:09.739Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("3ced66f6-8325-4509-802a-c6ffede8666b"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/3ced66f6-8325-4509-802a-c6ffede8666b"
region = 3ced66f6-8325-4509-802a-c6ffede8666b
|
| 19 | 2025-09-09T23:02:09.741Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "3ced66f6-8325-4509-802a-c6ffede8666b" "--data" "/data/regions/3ced66f6-8325-4509-802a-c6ffede8666b" "--block-size" "512" "--extent-size" "131072" "--extent-count" "16" "--encrypted"
region = 3ced66f6-8325-4509-802a-c6ffede8666b
|
| 20 | 2025-09-09T23:02:09.916Z | INFO | crucible-agent (dropshot): request completed
latency_us = 76
local_addr = [fd00:1122:3344:101::14]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:45621
req_id = ecbd2ab4-1a3c-4d4d-ba48-fd52b84390bc
response_code = 200
uri = /crucible/0/regions
|
| 21 | 2025-09-09T23:02:10.068Z | INFO | crucible-agent (worker): region files created ok
region = 3ced66f6-8325-4509-802a-c6ffede8666b
|
| 22 | 2025-09-09T23:02:10.068Z | INFO | crucible-agent (datafile): region 3ced66f6-8325-4509-802a-c6ffede8666b state: Requested -> Created
|
| 23 | 2025-09-09T23:02:10.068Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 24 | 2025-09-09T23:02:10.069Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-3ced66f6-8325-4509-802a-c6ffede8666b
|
| 25 | 2025-09-09T23:02:10.074Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-3ced66f6-8325-4509-802a-c6ffede8666b
|
| 26 | 2025-09-09T23:02:10.087Z | INFO | crucible-agent (worker): creating config property group
|
| 27 | 2025-09-09T23:02:10.089Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-3ced66f6-8325-4509-802a-c6ffede8666b
|
| 28 | 2025-09-09T23:02:10.090Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/3ced66f6-8325-4509-802a-c6ffede8666b
|
| 29 | 2025-09-09T23:02:10.098Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19000
|
| 30 | 2025-09-09T23:02:10.098Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14
|
| 31 | 2025-09-09T23:02:10.098Z | INFO | crucible-agent (worker): commit
|
| 32 | 2025-09-09T23:02:10.102Z | INFO | crucible-agent (worker): ok!
|
| 33 | 2025-09-09T23:02:10.106Z | INFO | crucible-agent (worker): SMF ok!
|
| 34 | 2025-09-09T23:02:10.418Z | INFO | crucible-agent (dropshot): request completed
latency_us = 48
local_addr = [fd00:1122:3344:101::14]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:45621
req_id = 63c1c0df-7e9c-40fb-a946-ee1926169a4b
response_code = 200
uri = /crucible/0/regions
|
| 35 | 2025-09-09T23:02:10.804Z | INFO | crucible-agent (dropshot): request completed
latency_us = 67
local_addr = [fd00:1122:3344:101::14]:32345
method = GET
remote_addr = [fd00:1122:3344:101::b]:45621
req_id = da99dc9e-6d97-4fe5-8b45-beee74217428
response_code = 200
uri = /crucible/0/regions/3ced66f6-8325-4509-802a-c6ffede8666b
|
| 36 | 2025-09-09T23:02:10.806Z | INFO | crucible-agent (datafile): path is "/data/regions/3ced66f6-8325-4509-802a-c6ffede8666b"
|
| 37 | 2025-09-09T23:02:10.818Z | INFO | crucible-agent (datafile): dataset is oxp_a5e1ea0b-7e8d-4c32-affe-b8fb952236ec/crucible/regions/3ced66f6-8325-4509-802a-c6ffede8666b
|
| 38 | 2025-09-09T23:02:10.830Z | INFO | crucible-agent (datafile): region 3ced66f6-8325-4509-802a-c6ffede8666b state: Created -> Tombstoned
|
| 39 | 2025-09-09T23:02:10.830Z | INFO | crucible-agent (worker): applying SMF actions before removal...
|
| 40 | 2025-09-09T23:02:10.830Z | INFO | crucible-agent (dropshot): request completed
latency_us = 23973
local_addr = [fd00:1122:3344:101::14]:32345
method = DELETE
remote_addr = [fd00:1122:3344:101::b]:45621
req_id = 01733a30-e814-404d-a93f-383583374385
response_code = 204
uri = /crucible/0/regions/3ced66f6-8325-4509-802a-c6ffede8666b
|
| 41 | 2025-09-09T23:02:10.831Z | INFO | crucible-agent (worker): disabling downstairs instance: downstairs-3ced66f6-8325-4509-802a-c6ffede8666b (instance states: (Some(Online), None))
|
| 42 | 2025-09-09T23:02:10.834Z | INFO | crucible-agent (dropshot): request completed
latency_us = 28
local_addr = [fd00:1122:3344:101::14]:32345
method = GET
remote_addr = [fd00:1122:3344:101::b]:45621
req_id = 777dee89-7792-4255-bb62-82bc4fc26e10
response_code = 200
uri = /crucible/0/regions/3ced66f6-8325-4509-802a-c6ffede8666b
|
| 43 | 2025-09-09T23:02:10.836Z | INFO | crucible-agent (worker): SMF ok!
|
| 44 | 2025-09-09T23:02:10.848Z | INFO | crucible-agent (worker): deleting zfs dataset "oxp_a5e1ea0b-7e8d-4c32-affe-b8fb952236ec/crucible/regions/3ced66f6-8325-4509-802a-c6ffede8666b"
region = 3ced66f6-8325-4509-802a-c6ffede8666b
|
| 45 | 2025-09-09T23:02:10.918Z | INFO | crucible-agent (datafile): region 3ced66f6-8325-4509-802a-c6ffede8666b state: Tombstoned -> Destroyed
|
| 46 | 2025-09-09T23:02:10.966Z | INFO | crucible-agent (dropshot): request completed
latency_us = 62
local_addr = [fd00:1122:3344:101::14]:32345
method = GET
remote_addr = [fd00:1122:3344:101::b]:45621
req_id = b1ca0ca4-05b0-4b57-8e8b-d3d61b899790
response_code = 200
uri = /crucible/0/regions/3ced66f6-8325-4509-802a-c6ffede8666b
|
| 47 | 2025-09-09T23:12:26.843Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::14]:32345
remote_addr = [fd00:1122:3344:101::b]:45231
|
| 48 | 2025-09-09T23:12:26.844Z | INFO | crucible-agent (datafile): region 4cd13338-b6dc-4095-b148-fcfd721a7020 state: Requested
|
| 49 | 2025-09-09T23:12:26.844Z | INFO | crucible-agent (dropshot): request completed
latency_us = 737
local_addr = [fd00:1122:3344:101::14]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:45231
req_id = 109459a3-d4e1-4cef-8a02-7c1cd5044cb4
response_code = 200
uri = /crucible/0/regions
|
| 50 | 2025-09-09T23:12:26.844Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 51 | 2025-09-09T23:12:26.857Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_a5e1ea0b-7e8d-4c32-affe-b8fb952236ec/crucible/regions/4cd13338-b6dc-4095-b148-fcfd721a7020
|
| 52 | 2025-09-09T23:12:26.857Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_a5e1ea0b-7e8d-4c32-affe-b8fb952236ec/crucible/regions/4cd13338-b6dc-4095-b148-fcfd721a7020
|
| 53 | 2025-09-09T23:12:26.915Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("4cd13338-b6dc-4095-b148-fcfd721a7020"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/4cd13338-b6dc-4095-b148-fcfd721a7020"
region = 4cd13338-b6dc-4095-b148-fcfd721a7020
|
| 54 | 2025-09-09T23:12:26.915Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "4cd13338-b6dc-4095-b148-fcfd721a7020" "--data" "/data/regions/4cd13338-b6dc-4095-b148-fcfd721a7020" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = 4cd13338-b6dc-4095-b148-fcfd721a7020
|
| 55 | 2025-09-09T23:12:27.039Z | INFO | crucible-agent (dropshot): request completed
latency_us = 97
local_addr = [fd00:1122:3344:101::14]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:45231
req_id = 637257ce-1475-4b8f-bace-6e7cba65c1c1
response_code = 200
uri = /crucible/0/regions
|
| 56 | 2025-09-09T23:12:27.188Z | INFO | crucible-agent (worker): region files created ok
region = 4cd13338-b6dc-4095-b148-fcfd721a7020
|
| 57 | 2025-09-09T23:12:27.188Z | INFO | crucible-agent (datafile): region 4cd13338-b6dc-4095-b148-fcfd721a7020 state: Requested -> Created
|
| 58 | 2025-09-09T23:12:27.188Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 59 | 2025-09-09T23:12:27.189Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-4cd13338-b6dc-4095-b148-fcfd721a7020
|
| 60 | 2025-09-09T23:12:27.200Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-4cd13338-b6dc-4095-b148-fcfd721a7020
|
| 61 | 2025-09-09T23:12:27.221Z | INFO | crucible-agent (worker): creating config property group
|
| 62 | 2025-09-09T23:12:27.224Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-4cd13338-b6dc-4095-b148-fcfd721a7020
|
| 63 | 2025-09-09T23:12:27.224Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/4cd13338-b6dc-4095-b148-fcfd721a7020
|
| 64 | 2025-09-09T23:12:27.224Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19000
|
| 65 | 2025-09-09T23:12:27.224Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14
|
| 66 | 2025-09-09T23:12:27.224Z | INFO | crucible-agent (worker): commit
|
| 67 | 2025-09-09T23:12:27.227Z | INFO | crucible-agent (worker): ok!
|
| 68 | 2025-09-09T23:12:27.231Z | INFO | crucible-agent (worker): SMF ok!
|
| 69 | 2025-09-09T23:12:27.501Z | INFO | crucible-agent (dropshot): request completed
latency_us = 47
local_addr = [fd00:1122:3344:101::14]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:45231
req_id = 66f83811-2bb6-4c19-8530-bd5300d87c8d
response_code = 200
uri = /crucible/0/regions
|
| 70 | 2025-09-09T23:12:33.933Z | INFO | crucible-agent (datafile): region ea83bbf1-6532-4dc3-81e9-297d564cd69a state: Requested
|
| 71 | 2025-09-09T23:12:33.933Z | INFO | crucible-agent (dropshot): request completed
latency_us = 229
local_addr = [fd00:1122:3344:101::14]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:45231
req_id = 7e4fc420-6406-425d-9e45-eba103984cfa
response_code = 200
uri = /crucible/0/regions
|
| 72 | 2025-09-09T23:12:33.933Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 73 | 2025-09-09T23:12:33.944Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_a5e1ea0b-7e8d-4c32-affe-b8fb952236ec/crucible/regions/ea83bbf1-6532-4dc3-81e9-297d564cd69a
|
| 74 | 2025-09-09T23:12:33.944Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_a5e1ea0b-7e8d-4c32-affe-b8fb952236ec/crucible/regions/ea83bbf1-6532-4dc3-81e9-297d564cd69a
|
| 75 | 2025-09-09T23:12:34.012Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("ea83bbf1-6532-4dc3-81e9-297d564cd69a"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/ea83bbf1-6532-4dc3-81e9-297d564cd69a"
region = ea83bbf1-6532-4dc3-81e9-297d564cd69a
|
| 76 | 2025-09-09T23:12:34.014Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "ea83bbf1-6532-4dc3-81e9-297d564cd69a" "--data" "/data/regions/ea83bbf1-6532-4dc3-81e9-297d564cd69a" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = ea83bbf1-6532-4dc3-81e9-297d564cd69a
|
| 77 | 2025-09-09T23:12:34.089Z | INFO | crucible-agent (worker): region files created ok
region = ea83bbf1-6532-4dc3-81e9-297d564cd69a
|
| 78 | 2025-09-09T23:12:34.089Z | INFO | crucible-agent (datafile): region ea83bbf1-6532-4dc3-81e9-297d564cd69a state: Requested -> Created
|
| 79 | 2025-09-09T23:12:34.089Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 80 | 2025-09-09T23:12:34.091Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-ea83bbf1-6532-4dc3-81e9-297d564cd69a
|
| 81 | 2025-09-09T23:12:34.099Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-ea83bbf1-6532-4dc3-81e9-297d564cd69a
|
| 82 | 2025-09-09T23:12:34.106Z | INFO | crucible-agent (worker): creating config property group
|
| 83 | 2025-09-09T23:12:34.109Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-ea83bbf1-6532-4dc3-81e9-297d564cd69a
|
| 84 | 2025-09-09T23:12:34.110Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/ea83bbf1-6532-4dc3-81e9-297d564cd69a
|
| 85 | 2025-09-09T23:12:34.110Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19001
|
| 86 | 2025-09-09T23:12:34.110Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14
|
| 87 | 2025-09-09T23:12:34.110Z | INFO | crucible-agent (worker): commit
|
| 88 | 2025-09-09T23:12:34.114Z | INFO | crucible-agent (worker): ok!
|
| 89 | 2025-09-09T23:12:34.119Z | INFO | crucible-agent (worker): SMF ok!
|
| 90 | 2025-09-09T23:12:34.174Z | INFO | crucible-agent (dropshot): request completed
latency_us = 64
local_addr = [fd00:1122:3344:101::14]:32345
method = POST
remote_addr = [fd00:1122:3344:101::b]:45231
req_id = 56e06f14-8c39-4ac1-84ff-2bbf5fb06ed4
response_code = 200
uri = /crucible/0/regions
|
| 91 | 2025-09-09T23:12:58.817Z | INFO | crucible-agent (dropshot): request completed
latency_us = 76
local_addr = [fd00:1122:3344:101::14]:32345
method = GET
remote_addr = [fd00:1122:3344:101::b]:45231
req_id = e990ad3e-07b3-4802-a207-4422c868989b
response_code = 200
uri = /crucible/0/regions/ea83bbf1-6532-4dc3-81e9-297d564cd69a
|
| 92 | 2025-09-09T23:12:58.820Z | INFO | crucible-agent (datafile): path is "/data/regions/ea83bbf1-6532-4dc3-81e9-297d564cd69a"
|
| 93 | 2025-09-09T23:12:58.835Z | INFO | crucible-agent (datafile): dataset is oxp_a5e1ea0b-7e8d-4c32-affe-b8fb952236ec/crucible/regions/ea83bbf1-6532-4dc3-81e9-297d564cd69a
|
| 94 | 2025-09-09T23:12:58.845Z | INFO | crucible-agent (datafile): region ea83bbf1-6532-4dc3-81e9-297d564cd69a state: Created -> Tombstoned
|
| 95 | 2025-09-09T23:12:58.845Z | INFO | crucible-agent (worker): applying SMF actions before removal...
|
| 96 | 2025-09-09T23:12:58.845Z | INFO | crucible-agent (dropshot): request completed
latency_us = 25406
local_addr = [fd00:1122:3344:101::14]:32345
method = DELETE
remote_addr = [fd00:1122:3344:101::b]:45231
req_id = c62fec42-c8b9-45a2-aab1-e8136a103ac7
response_code = 204
uri = /crucible/0/regions/ea83bbf1-6532-4dc3-81e9-297d564cd69a
|
| 97 | 2025-09-09T23:12:58.847Z | INFO | crucible-agent (worker): disabling downstairs instance: downstairs-ea83bbf1-6532-4dc3-81e9-297d564cd69a (instance states: (Some(Online), None))
|
| 98 | 2025-09-09T23:12:58.849Z | INFO | crucible-agent (dropshot): request completed
latency_us = 22
local_addr = [fd00:1122:3344:101::14]:32345
method = GET
remote_addr = [fd00:1122:3344:101::b]:45231
req_id = 71a80598-5d3c-48c3-9ed4-176ac6d4a26f
response_code = 200
uri = /crucible/0/regions/ea83bbf1-6532-4dc3-81e9-297d564cd69a
|
| 99 | 2025-09-09T23:12:58.858Z | INFO | crucible-agent (worker): SMF ok!
|
| 100 | 2025-09-09T23:12:58.869Z | INFO | crucible-agent (worker): deleting zfs dataset "oxp_a5e1ea0b-7e8d-4c32-affe-b8fb952236ec/crucible/regions/ea83bbf1-6532-4dc3-81e9-297d564cd69a"
region = ea83bbf1-6532-4dc3-81e9-297d564cd69a
|
| 101 | 2025-09-09T23:12:58.942Z | INFO | crucible-agent (datafile): region ea83bbf1-6532-4dc3-81e9-297d564cd69a state: Tombstoned -> Destroyed
|
| 102 | 2025-09-09T23:12:58.985Z | INFO | crucible-agent (dropshot): request completed
latency_us = 63
local_addr = [fd00:1122:3344:101::14]:32345
method = GET
remote_addr = [fd00:1122:3344:101::b]:45231
req_id = 4055985b-3677-482d-901a-000566c28cfb
response_code = 200
uri = /crucible/0/regions/ea83bbf1-6532-4dc3-81e9-297d564cd69a
|