| 1 | | [ Aug 17 05:18:53 Enabled. ] |
| 2 | | [ Aug 17 05:18:53 Rereading configuration. ] |
| 3 | | [ Aug 17 05:18:55 Rereading configuration. ] |
| 4 | | [ Aug 17 05:18:57 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_9ae0efee-a5e3-4544-b192-02596389e1f2/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ] |
| 5 | | note: configured to log to "/dev/stdout" |
| 6 | 2024-08-17T05:18:57.988Z | INFO | crucible-agent: dataset: "oxp_9ae0efee-a5e3-4544-b192-02596389e1f2/crucible"
|
| 7 | 2024-08-17T05:18:57.989Z | INFO | crucible-agent: listen IP: [fd00:1122:3344:101::13]:32345
|
| 8 | 2024-08-17T05:18:57.989Z | INFO | crucible-agent: SMF instance name downstairs_prefix: "downstairs"
|
| 9 | 2024-08-17T05:18:58.004Z | INFO | crucible-agent (datafile): Using conf_path:"/data/crucible.json"
|
| 10 | 2024-08-17T05:18:58.275Z | INFO | crucible-agent (dropshot): listening
local_addr = [fd00:1122:3344:101::13]:32345
|
| 11 | 2024-08-17T05:20:06.157Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::13]:32345
remote_addr = [fd00:1122:3344:101::a]:32808
|
| 12 | 2024-08-17T05:20:06.176Z | INFO | crucible-agent (datafile): region 61523ef9-ea95-4344-9216-6e97b222ddc1 state: Requested
|
| 13 | 2024-08-17T05:20:06.181Z | INFO | crucible-agent (worker): Region size:1073741824 reservation:1342177280 quota:3221225472
|
| 14 | 2024-08-17T05:20:06.185Z | INFO | crucible-agent (dropshot): request completed
latency_us = 13397
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:32808
req_id = 20aacade-7ae3-4efb-ad96-f148c5cc4daf
response_code = 200
uri = /crucible/0/regions
|
| 15 | 2024-08-17T05:20:06.192Z | INFO | crucible-agent (worker): zfs set reservation of 1342177280 for oxp_9ae0efee-a5e3-4544-b192-02596389e1f2/crucible/regions/61523ef9-ea95-4344-9216-6e97b222ddc1
|
| 16 | 2024-08-17T05:20:06.192Z | INFO | crucible-agent (worker): zfs set quota of 3221225472 for oxp_9ae0efee-a5e3-4544-b192-02596389e1f2/crucible/regions/61523ef9-ea95-4344-9216-6e97b222ddc1
|
| 17 | 2024-08-17T05:20:06.255Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("61523ef9-ea95-4344-9216-6e97b222ddc1"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/61523ef9-ea95-4344-9216-6e97b222ddc1"
region = 61523ef9-ea95-4344-9216-6e97b222ddc1
|
| 18 | 2024-08-17T05:20:06.255Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "61523ef9-ea95-4344-9216-6e97b222ddc1" "--data" "/data/regions/61523ef9-ea95-4344-9216-6e97b222ddc1" "--block-size" "512" "--extent-size" "131072" "--extent-count" "16" "--encrypted"
region = 61523ef9-ea95-4344-9216-6e97b222ddc1
|
| 19 | 2024-08-17T05:20:06.333Z | INFO | crucible-agent (dropshot): request completed
latency_us = 91
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:32808
req_id = 2b9d83c3-d015-4b62-a5b8-7cba60c0001a
response_code = 200
uri = /crucible/0/regions
|
| 20 | 2024-08-17T05:20:06.382Z | INFO | crucible-agent (worker): region files created ok
region = 61523ef9-ea95-4344-9216-6e97b222ddc1
|
| 21 | 2024-08-17T05:20:06.382Z | INFO | crucible-agent (datafile): region 61523ef9-ea95-4344-9216-6e97b222ddc1 state: Requested -> Created
|
| 22 | 2024-08-17T05:20:06.382Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 23 | 2024-08-17T05:20:06.382Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-61523ef9-ea95-4344-9216-6e97b222ddc1
|
| 24 | 2024-08-17T05:20:06.388Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-61523ef9-ea95-4344-9216-6e97b222ddc1
|
| 25 | 2024-08-17T05:20:06.405Z | INFO | crucible-agent (worker): creating config property group
|
| 26 | 2024-08-17T05:20:06.410Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-61523ef9-ea95-4344-9216-6e97b222ddc1
|
| 27 | 2024-08-17T05:20:06.417Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/61523ef9-ea95-4344-9216-6e97b222ddc1
|
| 28 | 2024-08-17T05:20:06.419Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19000
|
| 29 | 2024-08-17T05:20:06.419Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13
|
| 30 | 2024-08-17T05:20:06.419Z | INFO | crucible-agent (worker): commit
|
| 31 | 2024-08-17T05:20:06.424Z | INFO | crucible-agent (worker): ok!
|
| 32 | 2024-08-17T05:20:06.428Z | INFO | crucible-agent (worker): SMF ok!
|
| 33 | 2024-08-17T05:20:07.009Z | INFO | crucible-agent (dropshot): request completed
latency_us = 89
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:32808
req_id = 5a19f854-a516-4915-947b-7dd9c510b53b
response_code = 200
uri = /crucible/0/regions
|
| 34 | 2024-08-17T05:20:07.364Z | INFO | crucible-agent (dropshot): request completed
latency_us = 62
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:32808
req_id = d652c705-0b95-43d0-879c-a929a1229277
response_code = 200
uri = /crucible/0/regions/61523ef9-ea95-4344-9216-6e97b222ddc1
|
| 35 | 2024-08-17T05:20:07.366Z | INFO | crucible-agent (datafile): path is "/data/regions/61523ef9-ea95-4344-9216-6e97b222ddc1"
|
| 36 | 2024-08-17T05:20:07.375Z | INFO | crucible-agent (datafile): dataset is oxp_9ae0efee-a5e3-4544-b192-02596389e1f2/crucible/regions/61523ef9-ea95-4344-9216-6e97b222ddc1
|
| 37 | 2024-08-17T05:20:07.383Z | INFO | crucible-agent (datafile): region 61523ef9-ea95-4344-9216-6e97b222ddc1 state: Created -> Tombstoned
|
| 38 | 2024-08-17T05:20:07.384Z | INFO | crucible-agent (worker): applying SMF actions before removal...
|
| 39 | 2024-08-17T05:20:07.384Z | INFO | crucible-agent (dropshot): request completed
latency_us = 17415
local_addr = [fd00:1122:3344:101::13]:32345
method = DELETE
remote_addr = [fd00:1122:3344:101::a]:32808
req_id = 29ad90ec-47dd-43fe-b7d1-968453e54e06
response_code = 204
uri = /crucible/0/regions/61523ef9-ea95-4344-9216-6e97b222ddc1
|
| 40 | 2024-08-17T05:20:07.385Z | INFO | crucible-agent (worker): disabling downstairs instance: downstairs-61523ef9-ea95-4344-9216-6e97b222ddc1 (instance states: (Some(Online), None))
|
| 41 | 2024-08-17T05:20:07.387Z | INFO | crucible-agent (dropshot): request completed
latency_us = 480
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:32808
req_id = bfd5e11c-ea49-421d-818e-8ae242786874
response_code = 200
uri = /crucible/0/regions/61523ef9-ea95-4344-9216-6e97b222ddc1
|
| 42 | 2024-08-17T05:20:07.389Z | INFO | crucible-agent (worker): SMF ok!
|
| 43 | 2024-08-17T05:20:07.398Z | INFO | crucible-agent (worker): deleting zfs dataset "oxp_9ae0efee-a5e3-4544-b192-02596389e1f2/crucible/regions/61523ef9-ea95-4344-9216-6e97b222ddc1"
region = 61523ef9-ea95-4344-9216-6e97b222ddc1
|
| 44 | 2024-08-17T05:20:07.451Z | INFO | crucible-agent (datafile): region 61523ef9-ea95-4344-9216-6e97b222ddc1 state: Tombstoned -> Destroyed
|
| 45 | 2024-08-17T05:20:07.506Z | INFO | crucible-agent (dropshot): request completed
latency_us = 57
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:32808
req_id = 2fc7f947-4850-4d29-bf9c-8b0073309085
response_code = 200
uri = /crucible/0/regions/61523ef9-ea95-4344-9216-6e97b222ddc1
|
| 46 | 2024-08-17T05:20:09.082Z | INFO | crucible-agent (datafile): region 57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6 state: Requested
|
| 47 | 2024-08-17T05:20:09.082Z | INFO | crucible-agent (dropshot): request completed
latency_us = 214
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:32808
req_id = 4626e909-046e-4a89-b42e-4517c57b08ea
response_code = 200
uri = /crucible/0/regions
|
| 48 | 2024-08-17T05:20:09.082Z | INFO | crucible-agent (worker): Region size:2147483648 reservation:2684354560 quota:6442450944
|
| 49 | 2024-08-17T05:20:09.091Z | INFO | crucible-agent (worker): zfs set reservation of 2684354560 for oxp_9ae0efee-a5e3-4544-b192-02596389e1f2/crucible/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6
|
| 50 | 2024-08-17T05:20:09.091Z | INFO | crucible-agent (worker): zfs set quota of 6442450944 for oxp_9ae0efee-a5e3-4544-b192-02596389e1f2/crucible/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6
|
| 51 | 2024-08-17T05:20:09.140Z | INFO | crucible-agent (worker): creating region Region { id: RegionId("57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at "/data/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6"
region = 57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6
|
| 52 | 2024-08-17T05:20:09.142Z | INFO | crucible-agent (worker): downstairs create with: env -i "/opt/oxide/crucible/bin/crucible-downstairs" "create" "--uuid" "57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6" "--data" "/data/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6" "--block-size" "512" "--extent-size" "131072" "--extent-count" "32" "--encrypted"
region = 57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6
|
| 53 | 2024-08-17T05:20:09.213Z | INFO | crucible-agent (worker): region files created ok
region = 57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6
|
| 54 | 2024-08-17T05:20:09.213Z | INFO | crucible-agent (datafile): region 57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6 state: Requested -> Created
|
| 55 | 2024-08-17T05:20:09.213Z | INFO | crucible-agent (worker): applying SMF actions post create...
|
| 56 | 2024-08-17T05:20:09.214Z | INFO | crucible-agent (worker): creating missing downstairs instance downstairs-57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6
|
| 57 | 2024-08-17T05:20:09.217Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:downstairs-57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6
|
| 58 | 2024-08-17T05:20:09.225Z | INFO | crucible-agent (worker): creating config property group
|
| 59 | 2024-08-17T05:20:09.229Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:downstairs-57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6
|
| 60 | 2024-08-17T05:20:09.229Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6
|
| 61 | 2024-08-17T05:20:09.229Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19000
|
| 62 | 2024-08-17T05:20:09.229Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13
|
| 63 | 2024-08-17T05:20:09.229Z | INFO | crucible-agent (worker): commit
|
| 64 | 2024-08-17T05:20:09.234Z | INFO | crucible-agent (worker): ok!
|
| 65 | 2024-08-17T05:20:09.239Z | INFO | crucible-agent (worker): SMF ok!
|
| 66 | 2024-08-17T05:20:09.278Z | INFO | crucible-agent (dropshot): request completed
latency_us = 604
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:32808
req_id = 18d398ee-8441-4212-b779-67cc79331868
response_code = 200
uri = /crucible/0/regions
|
| 67 | 2024-08-17T05:25:08.122Z | INFO | crucible-agent (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::13]:32345
remote_addr = [fd00:1122:3344:101::a]:46453
|
| 68 | 2024-08-17T05:25:08.122Z | INFO | crucible-agent (dropshot): request completed
latency_us = 52
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:46453
req_id = d0931448-6699-4c78-b167-04b7e328e2a2
response_code = 200
uri = /crucible/0/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6
|
| 69 | 2024-08-17T05:25:08.128Z | INFO | crucible-agent (datafile): path is "/data/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6"
|
| 70 | 2024-08-17T05:25:08.137Z | INFO | crucible-agent (datafile): dataset is oxp_9ae0efee-a5e3-4544-b192-02596389e1f2/crucible/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6
|
| 71 | 2024-08-17T05:25:08.145Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_9ae0efee-a5e3-4544-b192-02596389e1f2/crucible/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6@28bb06e7-94ae-410d-bbac-c76370feda74
|
| 72 | 2024-08-17T05:25:08.145Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_9ae0efee-a5e3-4544-b192-02596389e1f2/crucible/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6", "28bb06e7-94ae-410d-bbac-c76370feda74"]
|
| 73 | 2024-08-17T05:25:08.157Z | INFO | crucible-agent (dropshot): request completed
latency_us = 31680
local_addr = [fd00:1122:3344:101::13]:32345
method = GET
remote_addr = [fd00:1122:3344:101::a]:46453
req_id = 259ab739-7654-4e65-9947-bfbe11209b70
response_code = 200
uri = /crucible/0/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6/snapshots/28bb06e7-94ae-410d-bbac-c76370feda74
|
| 74 | 2024-08-17T05:25:08.160Z | INFO | crucible-agent (datafile): path is "/data/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6"
|
| 75 | 2024-08-17T05:25:08.168Z | INFO | crucible-agent (datafile): dataset is oxp_9ae0efee-a5e3-4544-b192-02596389e1f2/crucible/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6
|
| 76 | 2024-08-17T05:25:08.176Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_9ae0efee-a5e3-4544-b192-02596389e1f2/crucible/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6@28bb06e7-94ae-410d-bbac-c76370feda74
|
| 77 | 2024-08-17T05:25:08.176Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_9ae0efee-a5e3-4544-b192-02596389e1f2/crucible/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6", "28bb06e7-94ae-410d-bbac-c76370feda74"]
|
| 78 | 2024-08-17T05:25:08.185Z | INFO | crucible-agent (datafile): requesting running snapshot 57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6-28bb06e7-94ae-410d-bbac-c76370feda74 state: Requested
|
| 79 | 2024-08-17T05:25:08.185Z | INFO | crucible-agent (worker): applying SMF actions for region 57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6 running snapshot 28bb06e7-94ae-410d-bbac-c76370feda74 (state Requested)...
|
| 80 | 2024-08-17T05:25:08.186Z | INFO | crucible-agent (dropshot): request completed
latency_us = 25861
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:46453
req_id = 2d05521c-e0eb-4897-aae9-b2eede17fd9b
response_code = 200
uri = /crucible/0/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6/snapshots/28bb06e7-94ae-410d-bbac-c76370feda74/run
|
| 81 | 2024-08-17T05:25:08.190Z | INFO | crucible-agent (worker): creating missing snapshot instance snapshot-57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6-28bb06e7-94ae-410d-bbac-c76370feda74
|
| 82 | 2024-08-17T05:25:08.197Z | INFO | crucible-agent (worker): ok, have svc:/oxide/crucible/downstairs:snapshot-57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6-28bb06e7-94ae-410d-bbac-c76370feda74
|
| 83 | 2024-08-17T05:25:08.208Z | INFO | crucible-agent (worker): reconfiguring svc:/oxide/crucible/downstairs:snapshot-57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6-28bb06e7-94ae-410d-bbac-c76370feda74
|
| 84 | 2024-08-17T05:25:08.208Z | INFO | crucible-agent (worker): ensure directory SCF_TYPE_ASTRING /data/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6/.zfs/snapshot/28bb06e7-94ae-410d-bbac-c76370feda74
|
| 85 | 2024-08-17T05:25:08.208Z | INFO | crucible-agent (worker): ensure port SCF_TYPE_COUNT 19001
|
| 86 | 2024-08-17T05:25:08.208Z | INFO | crucible-agent (worker): ensure mode SCF_TYPE_ASTRING ro
|
| 87 | 2024-08-17T05:25:08.208Z | INFO | crucible-agent (worker): ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13
|
| 88 | 2024-08-17T05:25:08.208Z | INFO | crucible-agent (worker): commit
|
| 89 | 2024-08-17T05:25:08.212Z | INFO | crucible-agent (worker): ok!
|
| 90 | 2024-08-17T05:25:08.216Z | INFO | crucible-agent (worker): SMF ok!
|
| 91 | 2024-08-17T05:25:08.216Z | INFO | crucible-agent (datafile): region 57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6 running snapshot 28bb06e7-94ae-410d-bbac-c76370feda74 state: Requested -> Created
|
| 92 | 2024-08-17T05:25:08.477Z | INFO | crucible-agent (datafile): path is "/data/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6"
|
| 93 | 2024-08-17T05:25:08.485Z | INFO | crucible-agent (datafile): dataset is oxp_9ae0efee-a5e3-4544-b192-02596389e1f2/crucible/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6
|
| 94 | 2024-08-17T05:25:08.494Z | INFO | crucible-agent (ZfsSnapshotInterface): snapshot is oxp_9ae0efee-a5e3-4544-b192-02596389e1f2/crucible/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6@28bb06e7-94ae-410d-bbac-c76370feda74
|
| 95 | 2024-08-17T05:25:08.494Z | INFO | crucible-agent (ZfsSnapshotInterface): parts is ["oxp_9ae0efee-a5e3-4544-b192-02596389e1f2/crucible/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6", "28bb06e7-94ae-410d-bbac-c76370feda74"]
|
| 96 | 2024-08-17T05:25:08.503Z | INFO | crucible-agent (dropshot): request completed
latency_us = 26836
local_addr = [fd00:1122:3344:101::13]:32345
method = POST
remote_addr = [fd00:1122:3344:101::a]:46453
req_id = fc6e690a-7208-4920-bcaa-99aa39eae0ff
response_code = 200
uri = /crucible/0/regions/57c5e72f-a0fc-4cb1-a8f2-ee4f72ab33a6/snapshots/28bb06e7-94ae-410d-bbac-c76370feda74/run
|