1 | 2024-02-01T09:00:33.908Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::18]:32345
remote_addr = [fd00:1122:3344:101::a]:57037
|
2 | 2024-02-01T09:00:33.940Z | INFO | crucible-agent (datafile): region 4516ea2b-d914-4c40-b86c-e70fd0818eae state: Requested
|
3 | 2024-02-01T09:00:33.943Z | INFO | crucible-agent (dropshot): request completed
latency_us = 2869
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:57037
req_id = 9c76b305-2c61-4b19-b32d-2a4698fdae93
response_code = 200
uri = /crucible/0/regions
|
4 | 2024-02-01T09:00:33.943Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
5 | 2024-02-01T09:00:33.952Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae
|
6 | 2024-02-01T09:00:33.952Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae
|
7 | 2024-02-01T09:00:34.002Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("4516ea2b-d914-4c40-b86c-e70fd0818eae"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at "/data/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae"
region = 4516ea2b-d914-4c40-b86c-e70fd0818eae
|
8 | 2024-02-01T09:00:34.130Z | INFO | crucible-agent (worker): region files created ok
region = 4516ea2b-d914-4c40-b86c-e70fd0818eae
|
9 | 2024-02-01T09:00:34.130Z | INFO | crucible-agent (datafile): region 4516ea2b-d914-4c40-b86c-e70fd0818eae state: Requested -> Created
|
10 | 2024-02-01T09:00:34.131Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
11 | 2024-02-01T09:00:34.132Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-4516ea2b-d914-4c40-b86c-e70fd0818eae
|
12 | 2024-02-01T09:00:34.139Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-4516ea2b-d914-4c40-b86c-e70fd0818eae
|
13 | 2024-02-01T09:00:34.150Z | INFO | crucible-agent (dropshot): request completed
latency_us = 79
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:57037
req_id = d73d913e-9e00-4a51-9c23-7366ead618f3
response_code = 200
uri = /crucible/0/regions
|
14 | 2024-02-01T09:00:34.157Z | INFO | crucible-agent (worker): creating config property group
|
15 | 2024-02-01T09:00:34.161Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-4516ea2b-d914-4c40-b86c-e70fd0818eae
|
16 | 2024-02-01T09:00:34.161Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae
|
17 | 2024-02-01T09:00:34.161Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19000
|
18 | 2024-02-01T09:00:34.161Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18
|
19 | 2024-02-01T09:00:34.161Z | INFO | crucible-agent (worker): commit
|
20 | 2024-02-01T09:00:34.168Z | INFO | crucible-agent (worker): ok!
|
21 | 2024-02-01T09:00:34.179Z | INFO | crucible-agent (worker): SMF ok!
|
22 | 2024-02-01T09:05:58.299Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::18]:32345
remote_addr = [fd00:1122:3344:101::a]:35829
|
23 | 2024-02-01T09:05:58.299Z | INFO | crucible-agent (datafile): region 8ebcd679-db0f-4a84-b34b-5d387d6d65d4 state: Requested
|
24 | 2024-02-01T09:05:58.300Z | INFO | crucible-agent (dropshot): request completed
latency_us = 776
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:35829
req_id = fdd29df8-9328-4a0a-8e08-2ba75e6c6e90
response_code = 200
uri = /crucible/0/regions
|
25 | 2024-02-01T09:05:58.300Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
26 | 2024-02-01T09:05:58.311Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/8ebcd679-db0f-4a84-b34b-5d387d6d65d4
|
27 | 2024-02-01T09:05:58.311Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/8ebcd679-db0f-4a84-b34b-5d387d6d65d4
|
28 | 2024-02-01T09:05:58.392Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("8ebcd679-db0f-4a84-b34b-5d387d6d65d4"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None } at "/data/regions/8ebcd679-db0f-4a84-b34b-5d387d6d65d4"
region = 8ebcd679-db0f-4a84-b34b-5d387d6d65d4
|
29 | 2024-02-01T09:05:58.477Z | INFO | crucible-agent (dropshot): request completed
latency_us = 145
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:35829
req_id = c84ca4da-1de6-4b56-a7ef-14917e1a6003
response_code = 200
uri = /crucible/0/regions
|
30 | 2024-02-01T09:05:58.527Z | INFO | crucible-agent (worker): region files created ok
region = 8ebcd679-db0f-4a84-b34b-5d387d6d65d4
|
31 | 2024-02-01T09:05:58.527Z | INFO | crucible-agent (datafile): region 8ebcd679-db0f-4a84-b34b-5d387d6d65d4 state: Requested -> Created
|
32 | 2024-02-01T09:05:58.527Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
33 | 2024-02-01T09:05:58.534Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-8ebcd679-db0f-4a84-b34b-5d387d6d65d4
|
34 | 2024-02-01T09:05:58.541Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-8ebcd679-db0f-4a84-b34b-5d387d6d65d4
|
35 | 2024-02-01T09:05:58.558Z | INFO | crucible-agent (worker): creating config property group
|
36 | 2024-02-01T09:05:58.565Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-8ebcd679-db0f-4a84-b34b-5d387d6d65d4
|
37 | 2024-02-01T09:05:58.566Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/8ebcd679-db0f-4a84-b34b-5d387d6d65d4
|
38 | 2024-02-01T09:05:58.566Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19001
|
39 | 2024-02-01T09:05:58.566Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18
|
40 | 2024-02-01T09:05:58.566Z | INFO | crucible-agent (worker): commit
|
41 | 2024-02-01T09:05:58.572Z | INFO | crucible-agent (worker): ok!
|
42 | 2024-02-01T09:05:58.584Z | INFO | crucible-agent (worker): SMF ok!
|
43 | 2024-02-01T09:05:59.116Z | INFO | crucible-agent (dropshot): request completed
latency_us = 80
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:35829
req_id = 847cbba9-9fcc-4154-9762-8b7076ec679c
response_code = 200
uri = /crucible/0/regions
|
44 | 2024-02-01T09:05:59.648Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::18]:32345
remote_addr = [fd00:1122:3344:101::a]:50240
|
45 | 2024-02-01T09:05:59.648Z | INFO | crucible-agent (dropshot): request completed
latency_us = 665
local_addr = [fd00:1122:3344:101::18]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:50240
req_id = 2fc1e56f-d148-4f16-bae0-cf94a6bd6efe
response_code = 200
uri = /crucible/0/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae
|
46 | 2024-02-01T09:05:59.651Z | INFO | crucible-agent (datafile): path is "/data/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae"
|
47 | 2024-02-01T09:05:59.659Z | INFO | crucible-agent (datafile): dataset is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae
|
48 | 2024-02-01T09:05:59.669Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae@2839f329-0315-4eef-84bf-18eb46bddaa5
|
49 | 2024-02-01T09:05:59.669Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae", "2839f329-0315-4eef-84bf-18eb46bddaa5"]
|
50 | 2024-02-01T09:05:59.678Z | INFO | crucible-agent (dropshot): request completed
latency_us = 29649
local_addr = [fd00:1122:3344:101::18]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:50240
req_id = 2f86edf0-bb8a-4f11-ba6d-be37e395963a
response_code = 200
uri = /crucible/0/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae/snapshots/2839f329-0315-4eef-84bf-18eb46bddaa5
|
51 | 2024-02-01T09:05:59.678Z | INFO | crucible-agent (datafile): path is "/data/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae"
|
52 | 2024-02-01T09:05:59.687Z | INFO | crucible-agent (datafile): dataset is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae
|
53 | 2024-02-01T09:05:59.696Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae@2839f329-0315-4eef-84bf-18eb46bddaa5
|
54 | 2024-02-01T09:05:59.696Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae", "2839f329-0315-4eef-84bf-18eb46bddaa5"]
|
55 | 2024-02-01T09:05:59.706Z | INFO | crucible-agent (datafile): requesting running snapshot 4516ea2b-d914-4c40-b86c-e70fd0818eae-2839f329-0315-4eef-84bf-18eb46bddaa5 state: Requested
|
56 | 2024-02-01T09:05:59.706Z | INFO | crucible-agent (worker): applying SMF actions for region 4516ea2b-d914-4c40-b86c-e70fd0818eae running snapshot 2839f329-0315-4eef-84bf-18eb46bddaa5 (state Requested)...
|
57 | 2024-02-01T09:05:59.706Z | INFO | crucible-agent (dropshot): request completed
latency_us = 27505
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:50240
req_id = 106c03bc-e585-49f7-a156-82409a6f35a7
response_code = 200
uri = /crucible/0/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae/snapshots/2839f329-0315-4eef-84bf-18eb46bddaa5/run
|
58 | 2024-02-01T09:05:59.708Z | INFO | crucible-agent (worker): creating missing snapshot instance snapshot-4516ea2b-d914-4c40-b86c-e70fd0818eae-2839f329-0315-4eef-84bf-18eb46bddaa5
|
59 | 2024-02-01T09:05:59.717Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:snapshot-4516ea2b-d914-4c40-b86c-e70fd0818eae-2839f329-0315-4eef-84bf-18eb46bddaa5
|
60 | 2024-02-01T09:05:59.734Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:snapshot-4516ea2b-d914-4c40-b86c-e70fd0818eae-2839f329-0315-4eef-84bf-18eb46bddaa5
|
61 | 2024-02-01T09:05:59.734Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae/.zfs/snapshot/2839f329-0315-4eef-84bf-18eb46bddaa5
|
62 | 2024-02-01T09:05:59.734Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19002
|
63 | 2024-02-01T09:05:59.734Z | INFO | crucible-agent (worker): ensure mode SCF_TYPE_ASTRING ro
|
64 | 2024-02-01T09:05:59.734Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18
|
65 | 2024-02-01T09:05:59.734Z | INFO | crucible-agent (worker): commit
|
66 | 2024-02-01T09:05:59.739Z | INFO | crucible-agent (worker): ok!
|
67 | 2024-02-01T09:05:59.748Z | INFO | crucible-agent (worker): SMF ok!
|
68 | 2024-02-01T09:05:59.748Z | INFO | crucible-agent (datafile): region 4516ea2b-d914-4c40-b86c-e70fd0818eae running snapshot 2839f329-0315-4eef-84bf-18eb46bddaa5 state: Requested -> Created
|
69 | 2024-02-01T09:06:02.315Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::18]:32345
remote_addr = [fd00:1122:3344:101::a]:33658
|
70 | 2024-02-01T09:06:02.347Z | INFO | crucible-agent (datafile): region 22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7 state: Requested
|
71 | 2024-02-01T09:06:02.347Z | INFO | crucible-agent (dropshot): request completed
latency_us = 214
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:33658
req_id = f4758631-ecd2-41e6-a9b2-bb9dbb6d089c
response_code = 200
uri = /crucible/0/regions
|
72 | 2024-02-01T09:06:02.347Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
73 | 2024-02-01T09:06:02.357Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7
|
74 | 2024-02-01T09:06:02.357Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7
|
75 | 2024-02-01T09:06:02.431Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None } at "/data/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7"
region = 22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7
|
76 | 2024-02-01T09:06:02.478Z | INFO | crucible-agent (dropshot): request completed
latency_us = 82
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:33658
req_id = 1a982ac8-fad1-4af4-880d-fa7e03cff3f1
response_code = 200
uri = /crucible/0/regions
|
77 | 2024-02-01T09:06:02.542Z | INFO | crucible-agent (worker): region files created ok
region = 22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7
|
78 | 2024-02-01T09:06:02.542Z | INFO | crucible-agent (datafile): region 22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7 state: Requested -> Created
|
79 | 2024-02-01T09:06:02.542Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
80 | 2024-02-01T09:06:02.543Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7
|
81 | 2024-02-01T09:06:02.549Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7
|
82 | 2024-02-01T09:06:02.561Z | INFO | crucible-agent (worker): creating config property group
|
83 | 2024-02-01T09:06:02.568Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7
|
84 | 2024-02-01T09:06:02.568Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7
|
85 | 2024-02-01T09:06:02.568Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19003
|
86 | 2024-02-01T09:06:02.568Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18
|
87 | 2024-02-01T09:06:02.569Z | INFO | crucible-agent (worker): commit
|
88 | 2024-02-01T09:06:02.576Z | INFO | crucible-agent (worker): ok!
|
89 | 2024-02-01T09:06:02.593Z | INFO | crucible-agent (worker): SMF ok!
|
90 | 2024-02-01T09:06:02.762Z | INFO | crucible-agent (dropshot): request completed
latency_us = 67
local_addr = [fd00:1122:3344:101::18]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:33658
req_id = a188ba43-ab50-4a8d-8c34-88593e6f7a50
response_code = 200
uri = /crucible/0/regions
|
91 | 2024-02-01T09:06:33.597Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::18]:32345
remote_addr = [fd00:1122:3344:101::a]:39501
|
92 | 2024-02-01T09:06:33.597Z | INFO | crucible-agent (dropshot): request completed
latency_us = 54
local_addr = [fd00:1122:3344:101::18]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:39501
req_id = 10b7a44c-d10b-449a-9182-d80273c8aa7e
response_code = 200
uri = /crucible/0/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7
|
93 | 2024-02-01T09:06:33.597Z | INFO | crucible-agent (datafile): path is "/data/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7"
|
94 | 2024-02-01T09:06:33.608Z | INFO | crucible-agent (datafile): dataset is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7
|
95 | 2024-02-01T09:06:33.616Z | INFO | crucible-agent (datafile): region 22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7 state: Created -> Tombstoned
|
96 | 2024-02-01T09:06:33.617Z | INFO | crucible-agent (worker): applying SMF actions before removal...
|
97 | 2024-02-01T09:06:33.617Z | INFO | crucible-agent (dropshot): request completed
latency_us = 19403
local_addr = [fd00:1122:3344:101::18]:32345
method = DELETE
remote_addr = [fd00:1122:3344:101::a]:39501
req_id = abe14eeb-38ed-407c-8ff1-c73c9aa88a83
response_code = 204
uri = /crucible/0/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7
|
98 | 2024-02-01T09:06:33.617Z | INFO | crucible-agent (dropshot): request completed
latency_us = 25
local_addr = [fd00:1122:3344:101::18]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:39501
req_id = 54b464c2-1825-4ac1-b3a0-026308d9b974
response_code = 200
uri = /crucible/0/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7
|
99 | 2024-02-01T09:06:33.618Z | INFO | crucible-agent (worker): disabling downstairs instance: downstairs-22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7 (instance states: (Some(Online), None))
|
100 | 2024-02-01T09:06:33.628Z | INFO | crucible-agent (worker): SMF ok!
|
101 | 2024-02-01T09:06:33.638Z | INFO | crucible-agent (worker): deleting zfs dataset "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7"
region = 22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7
|
102 | 2024-02-01T09:06:33.683Z | INFO | crucible-agent (datafile): region 22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7 state: Tombstoned -> Destroyed
|
103 | 2024-02-01T09:06:33.687Z | INFO | crucible-agent (dropshot): request completed
latency_us = 110
local_addr = [fd00:1122:3344:101::18]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:39501
req_id = 80a1a9bb-2166-4836-a669-735ab9d90a0f
response_code = 200
uri = /crucible/0/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7
|