| 1 | | [ Aug 17 21:55:12 Enabled. ] |
| 2 | | [ Aug 17 21:55:12 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] |
| 3 | | [ Aug 17 21:55:12 Method "start" exited with status 0. ] |
| 4 | | [ Aug 17 21:55:12 Rereading configuration. ] |
| 5 | | [ Aug 17 21:55:12 No 'refresh' method defined. Treating as :true. ] |
| 6 | | note: configured to log to "/dev/stdout" |
| 7 | 2022-08-17T21:55:12.767Z | INFO | SledAgent: Starting mg-ddm service
|
| 8 | 2022-08-17T21:55:12.768Z | INFO | SledAgent: Importing mg-ddm service
path = /opt/oxide/mg-ddm/pkg/ddm/manifest.xml
|
| 9 | 2022-08-17T21:55:12.877Z | INFO | SledAgent: Setting mg-ddm interface
interface = net0/linklocal
|
| 10 | 2022-08-17T21:55:12.886Z | INFO | SledAgent: Enabling mg-ddm service
|
| 11 | 2022-08-17T21:55:12.893Z | INFO | SledAgent: detecting (real or simulated) SP
|
| 12 | 2022-08-17T21:55:12.893Z | INFO | SledAgent: setting up bootstrap agent server
|
| 13 | 2022-08-17T21:55:12.893Z | INFO | SledAgent: Ensuring config directory exists
path = /var/oxide
|
| 14 | 2022-08-17T21:55:13.394Z | INFO | SledAgent (BootstrapAgentServer): Started listening
local_addr = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 15 | 2022-08-17T21:55:13.394Z | INFO | SledAgent (BootstrapAgent): bootstrap service initializing
server = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 16 | 2022-08-17T21:55:13.394Z | INFO | SledAgent: Sending prefix to ddmd for advertisement
prefix = Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, mask: 64 }
|
| 17 | 2022-08-17T21:55:13.395Z | INFO | SledAgent (RSS): Injecting RSS configuration: SetupServiceConfig {
rack_subnet: fd00:1122:3344:100::,
requests: [
HardcodedSledRequest {
datasets: [
DatasetEnsureBody {
id: 09a9a25f26024e2f963031af9c492c3e,
zpool_id: d462a7f7b62840fe80ff4e4189e2d62b,
dataset_kind: Crucible,
address: [fd00:1122:3344:101::6]:32345,
},
DatasetEnsureBody {
id: 2713b37a30434ed5aafff38200e45cfb,
zpool_id: e4b4dc87ab4649fba4b4d361ae214c03,
dataset_kind: Crucible,
address: [fd00:1122:3344:101::7]:32345,
},
DatasetEnsureBody {
id: ffd16cade5d5495e9c594312a3857d91,
zpool_id: f4b4dc87ab4649fba4b4d361ae214c03,
dataset_kind: Crucible,
address: [fd00:1122:3344:101::8]:32345,
},
DatasetEnsureBody {
id: 4d08fc193d5f4f6b9c48925f8eac7255,
zpool_id: d462a7f7b62840fe80ff4e4189e2d62b,
dataset_kind: CockroachDb {
all_addresses: [
[fd00:1122:3344:101::2]:32221,
],
},
address: [fd00:1122:3344:101::2]:32221,
},
DatasetEnsureBody {
id: a3505b41a592420b84f23d76bf0e0a81,
zpool_id: d462a7f7b62840fe80ff4e4189e2d62b,
dataset_kind: Clickhouse,
address: [fd00:1122:3344:101::5]:8123,
},
],
services: [
ServiceRequest {
id: e6bff1ff24fb49dca54ec6a350cd4d6c,
name: "nexus",
addresses: [
fd00:1122:3344:101::3,
],
gz_addresses: [],
service_type: Nexus {
internal_ip: fd00:1122:3344:101::3,
external_ip: 192.168.1.20,
},
},
ServiceRequest {
id: 1da65e5b210c4859a7d7200c1e659972,
name: "oximeter",
addresses: [
fd00:1122:3344:101::4,
],
gz_addresses: [],
service_type: Oximeter,
},
ServiceRequest {
id: a0fe5ebc92616f77acc1972481755789,
name: "dendrite",
addresses: [
fd00:1122:3344:101::9,
],
gz_addresses: [],
service_type: Dendrite {
asic: TofinoStub,
},
},
],
dns_services: [],
},
],
rack_secret_threshold: 1,
gateway: Gateway {
address: None,
mac: MacAddr6(
[
0,
13,
185,
84,
254,
228,
],
),
},
}
|
| 18 | 2022-08-17T21:55:13.395Z | INFO | SledAgent (RSS): RSS configuration has not been fully applied yet
|
| 19 | 2022-08-17T21:55:13.401Z | INFO | SledAgent (RSS): Received prefixes from ddmd
prefixes = {"fe80::8:20ff:fe9e:7b26": [Ipv6Prefix { addr: fd00:1122:3344:1::, mask: 64 }, Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, mask: 64 }, Ipv6Prefix { addr: fd00:1122:3344:101::, mask: 64 }]}
|
| 20 | 2022-08-17T21:55:13.402Z | INFO | SledAgent (RSS): Enough peers exist to enact RSS plan
|
| 21 | 2022-08-17T21:55:13.402Z | INFO | SledAgent (RSS): Creating new allocation plan
|
| 22 | 2022-08-17T21:55:13.402Z | INFO | SledAgent (RSS): dns_subnets: [
DnsSubnet {
subnet: Ipv6Subnet {
net: Ipv6Net(
Ipv6Network {
addr: fd00:1122:3344:1::,
prefix: 64,
},
),
},
},
]
|
| 23 | 2022-08-17T21:55:13.402Z | INFO | SledAgent (RSS): Serializing plan
|
| 24 | 2022-08-17T21:55:13.402Z | INFO | SledAgent (RSS): Creating plan for the sled at fdb0:18c0:4d0d:a02a::1
|
| 25 | 2022-08-17T21:55:13.402Z | INFO | SledAgent (RSS): Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1
|
| 26 | 2022-08-17T21:55:13.402Z | INFO | SledAgent (RSS): Plan serialized as: ["[fdb0:18c0:4d0d:9fb2::1]:12346".initialization_request]
id = "33c48143-1242-4008-9aa4-999a029687c4"
rack_id = "48cc8fbc-7888-4dab-bf28-8d74984cdc47"
["[fdb0:18c0:4d0d:9fb2::1]:12346".initialization_request.gateway]
mac = [0, 13, 185, 84, 254, 228]
["[fdb0:18c0:4d0d:9fb2::1]:12346".initialization_request.subnet]
net = "fd00:1122:3344:102::/64"
["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request]
dataset = []
dns_service = []
service = []
["[fdb0:18c0:4d0d:a02a::1]:12346".initialization_request]
id = "41a31165-763a-473c-a3f1-5c3a43824db0"
rack_id = "48cc8fbc-7888-4dab-bf28-8d74984cdc47"
["[fdb0:18c0:4d0d:a02a::1]:12346".initialization_request.gateway]
mac = [0, 13, 185, 84, 254, 228]
["[fdb0:18c0:4d0d:a02a::1]:12346".initialization_request.subnet]
net = "fd00:1122:3344:101::/64"
[["[fdb0:18c0:4d0d:a02a::1]:12346".services_request.dataset]]
address = "[fd00:1122:3344:101::6]:32345"
id = "09a9a25f-2602-4e2f-9630-31af9c492c3e"
zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b"
["[fdb0:18c0:4d0d:a02a::1]:12346".services_request.dataset.dataset_kind]
type = "crucible"
[["[fdb0:18c0:4d0d:a02a::1]:12346".services_request.dataset]]
address = "[fd00:1122:3344:101::7]:32345"
id = "2713b37a-3043-4ed5-aaff-f38200e45cfb"
zpool_id = "e4b4dc87-ab46-49fb-a4b4-d361ae214c03"
["[fdb0:18c0:4d0d:a02a::1]:12346".services_request.dataset.dataset_kind]
type = "crucible"
[["[fdb0:18c0:4d0d:a02a::1]:12346".services_request.dataset]]
address = "[fd00:1122:3344:101::8]:32345"
id = "ffd16cad-e5d5-495e-9c59-4312a3857d91"
zpool_id = "f4b4dc87-ab46-49fb-a4b4-d361ae214c03"
["[fdb0:18c0:4d0d:a02a::1]:12346".services_request.dataset.dataset_kind]
type = "crucible"
[["[fdb0:18c0:4d0d:a02a::1]:12346".services_request.dataset]]
address = "[fd00:1122:3344:101::2]:32221"
id = "4d08fc19-3d5f-4f6b-9c48-925f8eac7255"
zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b"
["[fdb0:18c0:4d0d:a02a::1]:12346".services_request.dataset.dataset_kind]
all_addresses = ["[fd00:1122:3344:101::2]:32221"]
type = "cockroach_db"
[["[fdb0:18c0:4d0d:a02a::1]:12346".services_request.dataset]]
address = "[fd00:1122:3344:101::5]:8123"
id = "a3505b41-a592-420b-84f2-3d76bf0e0a81"
zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b"
["[fdb0:18c0:4d0d:a02a::1]:12346".services_request.dataset.dataset_kind]
type = "clickhouse"
[["[fdb0:18c0:4d0d:a02a::1]:12346".services_request.dns_service]]
addresses = ["fd00:1122:3344:1::1"]
gz_addresses = ["fd00:1122:3344:1::2"]
id = "a4d8c1f2-9a46-4d3c-9841-d64efbd206e1"
name = "internal-dns"
["[fdb0:18c0:4d0d:a02a::1]:12346".services_request.dns_service.service_type]
dns_address = "[fd00:1122:3344:1::1]:53"
server_address = "[fd00:1122:3344:1::1]:5353"
type = "internal_dns"
[["[fdb0:18c0:4d0d:a02a::1]:12346".services_request.service]]
addresses = ["fd00:1122:3344:101::3"]
gz_addresses = []
id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"
name = "nexus"
["[fdb0:18c0:4d0d:a02a::1]:12346".services_request.service.service_type]
external_ip = "192.168.1.20"
internal_ip = "fd00:1122:3344:101::3"
type = "nexus"
[["[fdb0:18c0:4d0d:a02a::1]:12346".services_request.service]]
addresses = ["fd00:1122:3344:101::4"]
gz_addresses = []
id = "1da65e5b-210c-4859-a7d7-200c1e659972"
name = "oximeter"
["[fdb0:18c0:4d0d:a02a::1]:12346".services_request.service.service_type]
type = "oximeter"
[["[fdb0:18c0:4d0d:a02a::1]:12346".services_request.service]]
addresses = ["fd00:1122:3344:101::9"]
gz_addresses = []
id = "a0fe5ebc-9261-6f77-acc1-972481755789"
name = "dendrite"
["[fdb0:18c0:4d0d:a02a::1]:12346".services_request.service.service_type]
asic = "tofino_stub"
type = "dendrite"
|
| 27 | 2022-08-17T21:55:13.402Z | INFO | SledAgent (RSS): Plan written to storage
|
| 28 | 2022-08-17T21:55:13.402Z | INFO | SledAgent (RSS): Skipping rack secret creation (only one sled present)
|
| 29 | 2022-08-17T21:55:13.402Z | INFO | SledAgent (BootstrapAgentRssHandler): Received initialization request from RSS
request = SledAgentRequest { id: 41a31165-763a-473c-a3f1-5c3a43824db0, rack_id: 48cc8fbc-7888-4dab-bf28-8d74984cdc47, gateway: Gateway { address: None, mac: MacAddr6([0, 13, 185, 84, 254, 228]) }, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } }
target_sled = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 30 | 2022-08-17T21:55:13.402Z | INFO | SledAgent (BootstrapAgentRssHandler): Received initialization request from RSS
request = SledAgentRequest { id: 33c48143-1242-4008-9aa4-999a029687c4, rack_id: 48cc8fbc-7888-4dab-bf28-8d74984cdc47, gateway: Gateway { address: None, mac: MacAddr6([0, 13, 185, 84, 254, 228]) }, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:102::, prefix: 64 }) } }
target_sled = [fdb0:18c0:4d0d:9fb2::1]:12346
|
| 31 | 2022-08-17T21:55:13.403Z | INFO | SledAgent (BootstrapAgentServer): Accepted connection
remote_addr = [fdb0:18c0:4d0d:a02a::1]:47897
|
| 32 | 2022-08-17T21:55:13.403Z | INFO | SledAgent (BootstrapAgentServer): No SP available; proceeding without sprockets auth
remote_addr = [fdb0:18c0:4d0d:a02a::1]:47897
|
| 33 | 2022-08-17T21:55:13.403Z | INFO | SledAgent (BootstrapAgentRssHandler): No SP available; proceeding without sprockets auth
BootstrapAgentClient = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 34 | 2022-08-17T21:55:13.404Z | INFO | SledAgent (BootstrapAgentRssHandler): No SP available; proceeding without sprockets auth
BootstrapAgentClient = [fdb0:18c0:4d0d:9fb2::1]:12346
|
| 35 | 2022-08-17T21:55:13.404Z | INFO | SledAgent (BootstrapAgent): Loading Sled Agent: SledAgentRequest { id: 41a31165-763a-473c-a3f1-5c3a43824db0, rack_id: 48cc8fbc-7888-4dab-bf28-8d74984cdc47, gateway: Gateway { address: None, mac: MacAddr6([0, 13, 185, 84, 254, 228]) }, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } }
server = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 36 | 2022-08-17T21:55:13.404Z | INFO | SledAgent: setting up sled agent server
|
| 37 | 2022-08-17T21:55:13.404Z | INFO | SledAgent: created sled agent
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 38 | 2022-08-17T21:55:13.404Z | WARN | SledAgent (BootstrapAgentRssHandler): failed to start sled agent
error = ServerFailure("Sled agent already initialized")
|
| 39 | 2022-08-17T21:55:13.764Z | INFO | SledAgent (BootstrapAgentRssHandler): No SP available; proceeding without sprockets auth
BootstrapAgentClient = [fdb0:18c0:4d0d:9fb2::1]:12346
|
| 40 | 2022-08-17T21:55:13.764Z | WARN | SledAgent (BootstrapAgentRssHandler): failed to start sled agent
error = ServerFailure("Sled agent already initialized")
|
| 41 | 2022-08-17T21:55:13.896Z | INFO | SledAgent: updating xde driver configuration file for external IP workaround
conf_file = "/kernel/drv/xde.conf"
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 42 | 2022-08-17T21:55:13.938Z | INFO | SledAgent: using '[AddrObject { interface: "net0", name: "linklocal" }, AddrObject { interface: "net1", name: "linklocal" }]' as data links for xde driver
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 43 | 2022-08-17T21:55:14.067Z | INFO | SledAgent: Sled Agent upserting zpool to Storage Manager: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 44 | 2022-08-17T21:55:14.074Z | INFO | SledAgent: Sled Agent upserting zpool to Storage Manager: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 45 | 2022-08-17T21:55:14.082Z | INFO | SledAgent: Sled Agent upserting zpool to Storage Manager: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 46 | 2022-08-17T21:55:14.105Z | INFO | SledAgent (ServiceManager): No sled services found at /var/oxide/service.toml
|
| 47 | 2022-08-17T21:55:14.105Z | INFO | SledAgent (dropshot (SledAgent)): listening
local_addr = [fd00:1122:3344:101::1]:12345
|
| 48 | 2022-08-17T21:55:14.105Z | INFO | SledAgent (BootstrapAgent): Sled Agent loaded; recording configuration
server = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 49 | 2022-08-17T21:55:14.105Z | INFO | SledAgent (StorageManager): Storage manager processing zpool: ZpoolInfo {
name: "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b",
size: 10200547328,
allocated: 127488,
free: 10200419840,
health: Online,
}
|
| 50 | 2022-08-17T21:55:14.105Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 51 | 2022-08-17T21:55:14.109Z | INFO | SledAgent: Sending prefix to ddmd for advertisement
prefix = Ipv6Prefix { addr: fd00:1122:3344:101::, mask: 64 }
|
| 52 | 2022-08-17T21:55:14.109Z | INFO | SledAgent (BootstrapAgentServer): Connection closed
remote_addr = [fdb0:18c0:4d0d:a02a::1]:47897
|
| 53 | 2022-08-17T21:55:14.109Z | INFO | SledAgent (BootstrapAgentRssHandler): Peer agent initialized
peer = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 54 | 2022-08-17T21:55:14.109Z | INFO | SledAgent (BootstrapAgentRssHandler): Initialized sled agent
target_sled = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 55 | 2022-08-17T21:55:14.114Z | INFO | SledAgent (StorageManager): Storage manager processing zpool: ZpoolInfo {
name: "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03",
size: 10200547328,
allocated: 127488,
free: 10200419840,
health: Online,
}
|
| 56 | 2022-08-17T21:55:14.122Z | INFO | SledAgent (StorageManager): Storage manager processing zpool: ZpoolInfo {
name: "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03",
size: 10200547328,
allocated: 127488,
free: 10200419840,
health: Online,
}
|
| 57 | 2022-08-17T21:55:14.422Z | INFO | SledAgent (BootstrapAgentRssHandler): No SP available; proceeding without sprockets auth
BootstrapAgentClient = [fdb0:18c0:4d0d:9fb2::1]:12346
|
| 58 | 2022-08-17T21:55:14.422Z | WARN | SledAgent (BootstrapAgentRssHandler): failed to start sled agent
error = ServerFailure("Sled agent already initialized")
|
| 59 | 2022-08-17T21:55:15.373Z | INFO | SledAgent (BootstrapAgentRssHandler): No SP available; proceeding without sprockets auth
BootstrapAgentClient = [fdb0:18c0:4d0d:9fb2::1]:12346
|
| 60 | 2022-08-17T21:55:15.373Z | WARN | SledAgent (BootstrapAgentRssHandler): failed to start sled agent
error = ServerFailure("Sled agent already initialized")
|
| 61 | 2022-08-17T21:55:18.220Z | INFO | SledAgent (BootstrapAgentRssHandler): No SP available; proceeding without sprockets auth
BootstrapAgentClient = [fdb0:18c0:4d0d:9fb2::1]:12346
|
| 62 | 2022-08-17T21:55:20.054Z | WARN | SledAgent (BootstrapAgentRssHandler): failed to start sled agent
error = ServerFailure("Sled agent already initialized")
|
| 63 | 2022-08-17T21:55:24.679Z | INFO | SledAgent (BootstrapAgentRssHandler): No SP available; proceeding without sprockets auth
BootstrapAgentClient = [fdb0:18c0:4d0d:9fb2::1]:12346
|
| 64 | 2022-08-17T21:55:24.679Z | WARN | SledAgent (BootstrapAgentRssHandler): failed to start sled agent
error = ServerFailure("Sled agent already initialized")
|
| 65 | 2022-08-17T21:55:29.113Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 282.374548ms
|
| 66 | 2022-08-17T21:55:29.133Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 187.544723ms
|
| 67 | 2022-08-17T21:55:29.133Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 170.780111ms
|
| 68 | 2022-08-17T21:55:29.133Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 285.370543ms
|
| 69 | 2022-08-17T21:55:29.133Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 134.097859ms
|
| 70 | 2022-08-17T21:55:29.134Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 275.746131ms
|
| 71 | 2022-08-17T21:55:29.134Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 313.256497ms
|
| 72 | 2022-08-17T21:55:29.395Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 73 | 2022-08-17T21:55:29.750Z | INFO | SledAgent (BootstrapAgentRssHandler): No SP available; proceeding without sprockets auth
BootstrapAgentClient = [fdb0:18c0:4d0d:9fb2::1]:12346
|
| 74 | 2022-08-17T21:55:29.750Z | WARN | SledAgent (BootstrapAgentRssHandler): failed to start sled agent
error = ServerFailure("Sled agent already initialized")
|
| 75 | 2022-08-17T21:55:44.271Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 716.313238ms
|
| 76 | 2022-08-17T21:55:44.308Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 523.89096ms
|
| 77 | 2022-08-17T21:55:44.325Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 306.907938ms
|
| 78 | 2022-08-17T21:55:44.401Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 692.83995ms
|
| 79 | 2022-08-17T21:55:44.414Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 616.618912ms
|
| 80 | 2022-08-17T21:55:44.423Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 735.92177ms
|
| 81 | 2022-08-17T21:55:44.453Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 542.597825ms
|
| 82 | 2022-08-17T21:55:45.095Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 83 | 2022-08-17T21:55:59.635Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 1.343360475s
|
| 84 | 2022-08-17T21:55:59.835Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 643.24982ms
|
| 85 | 2022-08-17T21:55:59.990Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 783.145057ms
|
| 86 | 2022-08-17T21:55:59.999Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 803.042769ms
|
| 87 | 2022-08-17T21:56:00.034Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 1.102592269s
|
| 88 | 2022-08-17T21:56:00.098Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 1.356235146s
|
| 89 | 2022-08-17T21:56:00.165Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 704.084434ms
|
| 90 | 2022-08-17T21:56:01.455Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 91 | 2022-08-17T21:56:15.483Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.64044445s
|
| 92 | 2022-08-17T21:56:15.779Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.587577899s
|
| 93 | 2022-08-17T21:56:15.808Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.168312522s
|
| 94 | 2022-08-17T21:56:15.873Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 2.947047937s
|
| 95 | 2022-08-17T21:56:15.982Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 1.836910987s
|
| 96 | 2022-08-17T21:56:16.141Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 1.782793405s
|
| 97 | 2022-08-17T21:56:16.457Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 572.232851ms
|
| 98 | 2022-08-17T21:56:17.030Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 99 | 2022-08-17T21:56:31.981Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 3.978900653s
|
| 100 | 2022-08-17T21:56:32.034Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 556.334045ms
|
| 101 | 2022-08-17T21:56:32.128Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.124179524s
|
| 102 | 2022-08-17T21:56:32.371Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 3.66297391s
|
| 103 | 2022-08-17T21:56:32.591Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 104 | 2022-08-17T21:56:32.824Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 5.652314883s
|
| 105 | 2022-08-17T21:56:32.927Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 3.568103285s
|
| 106 | 2022-08-17T21:56:33.824Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 2.525749826s
|
| 107 | 2022-08-17T21:56:47.593Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 1.468787869s
|
| 108 | 2022-08-17T21:56:49.064Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 109 | 2022-08-17T21:56:49.256Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 8.624573107s
|
| 110 | 2022-08-17T21:56:50.964Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 10.123307245s
|
| 111 | 2022-08-17T21:56:51.038Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 4.074686711s
|
| 112 | 2022-08-17T21:56:51.355Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 7.348934512s
|
| 113 | 2022-08-17T21:56:51.500Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 11.392107905s
|
| 114 | 2022-08-17T21:56:53.480Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 5.358014925s
|
| 115 | 2022-08-17T21:57:04.067Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 941.596719ms
|
| 116 | 2022-08-17T21:57:05.009Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 117 | 2022-08-17T21:57:10.117Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 14.952335664s
|
| 118 | 2022-08-17T21:57:12.885Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 10.469584477s
|
| 119 | 2022-08-17T21:57:13.708Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 15.769283561s
|
| 120 | 2022-08-17T21:57:13.842Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 23.717419175s
|
| 121 | 2022-08-17T21:57:16.092Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 19.530332094s
|
| 122 | 2022-08-17T21:57:17.897Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 18.307336144s
|
| 123 | 2022-08-17T21:57:20.013Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 1.431940278s
|
| 124 | 2022-08-17T21:57:21.445Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 125 | 2022-08-17T21:57:36.448Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 1.323879013s
|
| 126 | 2022-08-17T21:57:37.774Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 127 | 2022-08-17T21:57:38.359Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 30.252396398s
|
| 128 | 2022-08-17T21:57:40.073Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 47.892141742s
|
| 129 | 2022-08-17T21:57:44.484Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 24.733502915s
|
| 130 | 2022-08-17T21:57:50.628Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 21.070584882s
|
| 131 | 2022-08-17T21:57:51.209Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 34.680508363s
|
| 132 | 2022-08-17T21:57:52.564Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 39.750289708s
|
| 133 | 2022-08-17T21:57:52.777Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 1.382054797s
|
| 134 | 2022-08-17T21:57:54.160Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 135 | 2022-08-17T21:58:09.163Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 584.147859ms
|
| 136 | 2022-08-17T21:58:09.748Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 137 | 2022-08-17T21:58:23.616Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 51.932202207s
|
| 138 | 2022-08-17T21:58:24.222Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 85.245694001s
|
| 139 | 2022-08-17T21:58:24.751Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 962.130952ms
|
| 140 | 2022-08-17T21:58:25.714Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 141 | 2022-08-17T21:58:26.704Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 58.274725471s
|
| 142 | 2022-08-17T21:58:40.717Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 668.553585ms
|
| 143 | 2022-08-17T21:58:40.894Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 36.165543188s
|
| 144 | 2022-08-17T21:58:41.386Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 145 | 2022-08-17T21:58:42.971Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 33.33504975s
|
| 146 | 2022-08-17T21:58:47.318Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 69.272072433s
|
| 147 | 2022-08-17T21:58:56.389Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 1.265547404s
|
| 148 | 2022-08-17T21:58:57.655Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 149 | 2022-08-17T21:59:01.063Z | WARN | SledAgent (BootstrapAgentRssHandler): failed to start sled agent
error = Connect { addr: [fdb0:18c0:4d0d:9fb2::1]:12346, err: Os { code: 148, kind: HostUnreachable, message: "No route to host" } }
|
| 150 | 2022-08-17T21:59:12.659Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 1.275054309s
|
| 151 | 2022-08-17T21:59:13.935Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 152 | 2022-08-17T21:59:28.939Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 1.102618955s
|
| 153 | 2022-08-17T21:59:30.043Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 154 | 2022-08-17T21:59:30.554Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 165.382271051s
|
| 155 | 2022-08-17T21:59:31.309Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 178.506791782s
|
| 156 | 2022-08-17T21:59:32.063Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 87.697750312s
|
| 157 | 2022-08-17T21:59:39.983Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 98.300050447s
|
| 158 | 2022-08-17T21:59:45.046Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 515.731418ms
|
| 159 | 2022-08-17T21:59:45.562Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 160 | 2022-08-17T22:00:00.567Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 1.196480127s
|
| 161 | 2022-08-17T22:00:01.765Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 162 | 2022-08-17T22:00:04.471Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 167.388431071s
|
| 163 | 2022-08-17T22:00:11.595Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 71.179360626s
|
| 164 | 2022-08-17T22:00:16.768Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 932.354948ms
|
| 165 | 2022-08-17T22:00:17.702Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 166 | 2022-08-17T22:00:32.705Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 1.394018904s
|
| 167 | 2022-08-17T22:00:34.100Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 168 | 2022-08-17T22:00:49.103Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 907.963831ms
|
| 169 | 2022-08-17T22:00:50.012Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 170 | 2022-08-17T22:01:05.015Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 1.015699622s
|
| 171 | 2022-08-17T22:01:06.032Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 172 | 2022-08-17T22:01:14.765Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 334.524778591s
|
| 173 | 2022-08-17T22:01:21.036Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 1.318336131s
|
| 174 | 2022-08-17T22:01:22.355Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 175 | 2022-08-17T22:01:33.287Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 225.291504593s
|
| 176 | 2022-08-17T22:01:37.358Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 1.209781953s
|
| 177 | 2022-08-17T22:01:37.777Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 163.490950644s
|
| 178 | 2022-08-17T22:01:38.568Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 179 | 2022-08-17T22:01:53.572Z | WARN | SledAgent: failed to notify nexus about sled agent: request timed out, will retry in 941.866595ms
|
| 180 | 2022-08-17T22:01:54.515Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|