1 | | [ Aug 17 20:58:07 Enabled. ] |
2 | | [ Aug 17 20:58:07 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] |
3 | | [ Aug 17 20:58:07 Method "start" exited with status 0. ] |
4 | | [ Aug 17 20:58:07 Rereading configuration. ] |
5 | | [ Aug 17 20:58:07 No 'refresh' method defined. Treating as :true. ] |
6 | | note: configured to log to "/dev/stdout" |
7 | 2022-08-17T20:58:07.698Z | INFO | SledAgent: Starting mg-ddm service
|
8 | 2022-08-17T20:58:07.698Z | INFO | SledAgent: Importing mg-ddm service
path = /opt/oxide/mg-ddm/pkg/ddm/manifest.xml
|
9 | 2022-08-17T20:58:07.804Z | INFO | SledAgent: Setting mg-ddm interface
interface = net0/linklocal
|
10 | 2022-08-17T20:58:07.812Z | INFO | SledAgent: Enabling mg-ddm service
|
11 | 2022-08-17T20:58:07.819Z | INFO | SledAgent: detecting (real or simulated) SP
|
12 | 2022-08-17T20:58:07.819Z | INFO | SledAgent: setting up bootstrap agent server
|
13 | 2022-08-17T20:58:07.819Z | INFO | SledAgent: Ensuring config directory exists
path = /var/oxide
|
14 | 2022-08-17T20:58:08.322Z | INFO | SledAgent (BootstrapAgentServer): Started listening
local_addr = [fdb0:18c0:4d0d:9fb2::1]:12346
|
15 | 2022-08-17T20:58:08.322Z | INFO | SledAgent (BootstrapAgent): bootstrap service initializing
server = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
16 | 2022-08-17T20:58:08.322Z | INFO | SledAgent: Sending prefix to ddmd for advertisement
prefix = Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, mask: 64 }
|
17 | 2022-08-17T20:58:08.323Z | INFO | SledAgent (RSS): Injecting RSS configuration: SetupServiceConfig {
rack_subnet: fd00:1122:3344:100::,
requests: [
HardcodedSledRequest {
datasets: [
DatasetEnsureBody {
id: 09a9a25f26024e2f963031af9c492c3e,
zpool_id: d462a7f7b62840fe80ff4e4189e2d62b,
dataset_kind: Crucible,
address: [fd00:1122:3344:101::6]:32345,
},
DatasetEnsureBody {
id: 2713b37a30434ed5aafff38200e45cfb,
zpool_id: e4b4dc87ab4649fba4b4d361ae214c03,
dataset_kind: Crucible,
address: [fd00:1122:3344:101::7]:32345,
},
DatasetEnsureBody {
id: ffd16cade5d5495e9c594312a3857d91,
zpool_id: f4b4dc87ab4649fba4b4d361ae214c03,
dataset_kind: Crucible,
address: [fd00:1122:3344:101::8]:32345,
},
DatasetEnsureBody {
id: 4d08fc193d5f4f6b9c48925f8eac7255,
zpool_id: d462a7f7b62840fe80ff4e4189e2d62b,
dataset_kind: CockroachDb {
all_addresses: [
[fd00:1122:3344:101::2]:32221,
],
},
address: [fd00:1122:3344:101::2]:32221,
},
DatasetEnsureBody {
id: a3505b41a592420b84f23d76bf0e0a81,
zpool_id: d462a7f7b62840fe80ff4e4189e2d62b,
dataset_kind: Clickhouse,
address: [fd00:1122:3344:101::5]:8123,
},
],
services: [
ServiceRequest {
id: e6bff1ff24fb49dca54ec6a350cd4d6c,
name: "nexus",
addresses: [
fd00:1122:3344:101::3,
],
gz_addresses: [],
service_type: Nexus {
internal_ip: fd00:1122:3344:101::3,
external_ip: 192.168.1.20,
},
},
ServiceRequest {
id: 1da65e5b210c4859a7d7200c1e659972,
name: "oximeter",
addresses: [
fd00:1122:3344:101::4,
],
gz_addresses: [],
service_type: Oximeter,
},
ServiceRequest {
id: a0fe5ebc92616f77acc1972481755789,
name: "dendrite",
addresses: [
fd00:1122:3344:101::9,
],
gz_addresses: [],
service_type: Dendrite {
asic: TofinoStub,
},
},
],
dns_services: [],
},
],
rack_secret_threshold: 1,
gateway: Gateway {
address: None,
mac: MacAddr6(
[
0,
13,
185,
84,
254,
228,
],
),
},
}
|
18 | 2022-08-17T20:58:08.323Z | INFO | SledAgent (RSS): RSS configuration has not been fully applied yet
|
19 | 2022-08-17T20:58:08.329Z | INFO | SledAgent (RSS): Received prefixes from ddmd
prefixes = {}
|
20 | 2022-08-17T20:58:08.329Z | INFO | SledAgent (RSS): Enough peers exist to enact RSS plan
|
21 | 2022-08-17T20:58:08.330Z | INFO | SledAgent (RSS): Creating new allocation plan
|
22 | 2022-08-17T20:58:08.330Z | INFO | SledAgent (RSS): dns_subnets: [
DnsSubnet {
subnet: Ipv6Subnet {
net: Ipv6Net(
Ipv6Network {
addr: fd00:1122:3344:1::,
prefix: 64,
},
),
},
},
]
|
23 | 2022-08-17T20:58:08.330Z | INFO | SledAgent (RSS): Serializing plan
|
24 | 2022-08-17T20:58:08.330Z | INFO | SledAgent (RSS): Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1
|
25 | 2022-08-17T20:58:08.330Z | INFO | SledAgent (RSS): Plan serialized as: ["[fdb0:18c0:4d0d:9fb2::1]:12346".initialization_request]
id = "5e049c3e-9621-48f2-a10c-7c4c53ae41a3"
rack_id = "7de63f8e-63a6-403e-ae13-3cdc2f407ab8"
["[fdb0:18c0:4d0d:9fb2::1]:12346".initialization_request.gateway]
mac = [0, 13, 185, 84, 254, 228]
["[fdb0:18c0:4d0d:9fb2::1]:12346".initialization_request.subnet]
net = "fd00:1122:3344:101::/64"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset]]
address = "[fd00:1122:3344:101::6]:32345"
id = "09a9a25f-2602-4e2f-9630-31af9c492c3e"
zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b"
["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset.dataset_kind]
type = "crucible"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset]]
address = "[fd00:1122:3344:101::7]:32345"
id = "2713b37a-3043-4ed5-aaff-f38200e45cfb"
zpool_id = "e4b4dc87-ab46-49fb-a4b4-d361ae214c03"
["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset.dataset_kind]
type = "crucible"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset]]
address = "[fd00:1122:3344:101::8]:32345"
id = "ffd16cad-e5d5-495e-9c59-4312a3857d91"
zpool_id = "f4b4dc87-ab46-49fb-a4b4-d361ae214c03"
["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset.dataset_kind]
type = "crucible"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset]]
address = "[fd00:1122:3344:101::2]:32221"
id = "4d08fc19-3d5f-4f6b-9c48-925f8eac7255"
zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b"
["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset.dataset_kind]
all_addresses = ["[fd00:1122:3344:101::2]:32221"]
type = "cockroach_db"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset]]
address = "[fd00:1122:3344:101::5]:8123"
id = "a3505b41-a592-420b-84f2-3d76bf0e0a81"
zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b"
["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset.dataset_kind]
type = "clickhouse"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dns_service]]
addresses = ["fd00:1122:3344:1::1"]
gz_addresses = ["fd00:1122:3344:1::2"]
id = "4b51d2cc-ccf1-49ea-b3bb-0463bb1cff8b"
name = "internal-dns"
["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dns_service.service_type]
dns_address = "[fd00:1122:3344:1::1]:53"
server_address = "[fd00:1122:3344:1::1]:5353"
type = "internal_dns"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.service]]
addresses = ["fd00:1122:3344:101::3"]
gz_addresses = []
id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"
name = "nexus"
["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.service.service_type]
external_ip = "192.168.1.20"
internal_ip = "fd00:1122:3344:101::3"
type = "nexus"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.service]]
addresses = ["fd00:1122:3344:101::4"]
gz_addresses = []
id = "1da65e5b-210c-4859-a7d7-200c1e659972"
name = "oximeter"
["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.service.service_type]
type = "oximeter"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.service]]
addresses = ["fd00:1122:3344:101::9"]
gz_addresses = []
id = "a0fe5ebc-9261-6f77-acc1-972481755789"
name = "dendrite"
["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.service.service_type]
asic = "tofino_stub"
type = "dendrite"
|
26 | 2022-08-17T20:58:08.330Z | INFO | SledAgent (RSS): Plan written to storage
|
27 | 2022-08-17T20:58:08.330Z | INFO | SledAgent (RSS): Skipping rack secret creation (only one sled present)
|
28 | 2022-08-17T20:58:08.330Z | INFO | SledAgent (BootstrapAgentRssHandler): Received initialization request from RSS
request = SledAgentRequest { id: 5e049c3e-9621-48f2-a10c-7c4c53ae41a3, rack_id: 7de63f8e-63a6-403e-ae13-3cdc2f407ab8, gateway: Gateway { address: None, mac: MacAddr6([0, 13, 185, 84, 254, 228]) }, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } }
target_sled = [fdb0:18c0:4d0d:9fb2::1]:12346
|
29 | 2022-08-17T20:58:08.330Z | INFO | SledAgent (BootstrapAgentServer): Accepted connection
remote_addr = [fdb0:18c0:4d0d:9fb2::1]:45495
|
30 | 2022-08-17T20:58:08.330Z | INFO | SledAgent (BootstrapAgentRssHandler): No SP available; proceeding without sprockets auth
BootstrapAgentClient = [fdb0:18c0:4d0d:9fb2::1]:12346
|
31 | 2022-08-17T20:58:08.330Z | INFO | SledAgent (BootstrapAgentServer): No SP available; proceeding without sprockets auth
remote_addr = [fdb0:18c0:4d0d:9fb2::1]:45495
|
32 | 2022-08-17T20:58:08.330Z | INFO | SledAgent (BootstrapAgent): Loading Sled Agent: SledAgentRequest { id: 5e049c3e-9621-48f2-a10c-7c4c53ae41a3, rack_id: 7de63f8e-63a6-403e-ae13-3cdc2f407ab8, gateway: Gateway { address: None, mac: MacAddr6([0, 13, 185, 84, 254, 228]) }, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } }
server = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
33 | 2022-08-17T20:58:08.330Z | INFO | SledAgent: setting up sled agent server
|
34 | 2022-08-17T20:58:08.330Z | INFO | SledAgent: created sled agent
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
35 | 2022-08-17T20:58:08.822Z | INFO | SledAgent: updating xde driver configuration file for external IP workaround
conf_file = "/kernel/drv/xde.conf"
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
36 | 2022-08-17T20:58:08.865Z | INFO | SledAgent: using '[AddrObject { interface: "net0", name: "linklocal" }, AddrObject { interface: "net1", name: "linklocal" }]' as data links for xde driver
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
37 | 2022-08-17T20:58:08.992Z | INFO | SledAgent: Sled Agent upserting zpool to Storage Manager: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
38 | 2022-08-17T20:58:08.999Z | INFO | SledAgent: Sled Agent upserting zpool to Storage Manager: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
39 | 2022-08-17T20:58:09.007Z | INFO | SledAgent: Sled Agent upserting zpool to Storage Manager: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
40 | 2022-08-17T20:58:09.031Z | INFO | SledAgent (ServiceManager): No sled services found at /var/oxide/service.toml
|
41 | 2022-08-17T20:58:09.031Z | INFO | SledAgent (dropshot (SledAgent)): listening
local_addr = [fd00:1122:3344:101::1]:12345
|
42 | 2022-08-17T20:58:09.031Z | INFO | SledAgent (BootstrapAgent): Sled Agent loaded; recording configuration
server = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
43 | 2022-08-17T20:58:09.031Z | INFO | SledAgent (StorageManager): Storage manager processing zpool: ZpoolInfo {
name: "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b",
size: 10200547328,
allocated: 127488,
free: 10200419840,
health: Online,
}
|
44 | 2022-08-17T20:58:09.031Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
45 | 2022-08-17T20:58:09.034Z | INFO | SledAgent: Sending prefix to ddmd for advertisement
prefix = Ipv6Prefix { addr: fd00:1122:3344:101::, mask: 64 }
|
46 | 2022-08-17T20:58:09.034Z | INFO | SledAgent (BootstrapAgentServer): Connection closed
remote_addr = [fdb0:18c0:4d0d:9fb2::1]:45495
|
47 | 2022-08-17T20:58:09.034Z | INFO | SledAgent (BootstrapAgentRssHandler): Peer agent initialized
peer = [fdb0:18c0:4d0d:9fb2::1]:12346
|
48 | 2022-08-17T20:58:09.034Z | INFO | SledAgent (BootstrapAgentRssHandler): Initialized sled agent
target_sled = [fdb0:18c0:4d0d:9fb2::1]:12346
|
49 | 2022-08-17T20:58:09.035Z | WARN | SledAgent: failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 190.483085ms
|
50 | 2022-08-17T20:58:09.039Z | INFO | SledAgent (StorageManager): Storage manager processing zpool: ZpoolInfo {
name: "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03",
size: 10200547328,
allocated: 127488,
free: 10200419840,
health: Online,
}
|
51 | 2022-08-17T20:58:09.043Z | INFO | SledAgent (RSS): sending service requests...
|
52 | 2022-08-17T20:58:09.043Z | INFO | SledAgent (RSS): initializing sled services: [ServiceRequest { id: 4b51d2cc-ccf1-49ea-b3bb-0463bb1cff8b, name: "internal-dns", addresses: [fd00:1122:3344:1::1], gz_addresses: [fd00:1122:3344:1::2], service_type: InternalDns { server_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53 } }]
|
53 | 2022-08-17T20:58:09.043Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:45829
|
54 | 2022-08-17T20:58:09.043Z | INFO | SledAgent (ServiceManager): Ensuring service is initialized: ServiceRequest { id: 4b51d2cc-ccf1-49ea-b3bb-0463bb1cff8b, name: "internal-dns", addresses: [fd00:1122:3344:1::1], gz_addresses: [fd00:1122:3344:1::2], service_type: InternalDns { server_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53 } }
|
55 | 2022-08-17T20:58:09.043Z | INFO | SledAgent (ServiceManager): Service internal-dns does not yet exist
|
56 | 2022-08-17T20:58:09.048Z | INFO | SledAgent (StorageManager): Storage manager processing zpool: ZpoolInfo {
name: "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03",
size: 10200547328,
allocated: 563712,
free: 10199983616,
health: Online,
}
|
57 | 2022-08-17T20:58:09.057Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 160.848712ms
|
58 | 2022-08-17T20:58:09.057Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 260.118052ms
|
59 | 2022-08-17T20:58:09.057Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 271.957022ms
|
60 | 2022-08-17T20:58:09.057Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 181.815006ms
|
61 | 2022-08-17T20:58:09.057Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 288.702761ms
|
62 | 2022-08-17T20:58:09.057Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 303.389995ms
|
63 | 2022-08-17T20:58:09.059Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal-dns
|
64 | 2022-08-17T20:58:09.075Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal-dns
|
65 | 2022-08-17T20:58:09.220Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 373.53334ms
|
66 | 2022-08-17T20:58:09.226Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
67 | 2022-08-17T20:58:09.226Z | WARN | SledAgent: failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 356.734703ms
|
68 | 2022-08-17T20:58:09.241Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 430.482854ms
|
69 | 2022-08-17T20:58:09.319Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 608.014163ms
|
70 | 2022-08-17T20:58:09.330Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 569.162412ms
|
71 | 2022-08-17T20:58:09.348Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 710.463436ms
|
72 | 2022-08-17T20:58:09.362Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 701.899657ms
|
73 | 2022-08-17T20:58:09.584Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
74 | 2022-08-17T20:58:09.584Z | WARN | SledAgent: failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 582.824299ms
|
75 | 2022-08-17T20:58:09.595Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 555.565087ms
|
76 | 2022-08-17T20:58:09.673Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 1.056384132s
|
77 | 2022-08-17T20:58:09.900Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.079211077s
|
78 | 2022-08-17T20:58:09.929Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 1.448080707s
|
79 | 2022-08-17T20:58:10.060Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 1.252593132s
|
80 | 2022-08-17T20:58:10.065Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 870.264992ms
|
81 | 2022-08-17T20:58:10.152Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.046919268s
|
82 | 2022-08-17T20:58:10.155Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_internal-dns
|
83 | 2022-08-17T20:58:10.169Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
84 | 2022-08-17T20:58:10.169Z | WARN | SledAgent: failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 773.978468ms
|
85 | 2022-08-17T20:58:10.731Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 1.167801443s
|
86 | 2022-08-17T20:58:10.936Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.869552801s
|
87 | 2022-08-17T20:58:10.944Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
88 | 2022-08-17T20:58:10.945Z | WARN | SledAgent: failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 913.216395ms
|
89 | 2022-08-17T20:58:10.981Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.747880958s
|
90 | 2022-08-17T20:58:11.314Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 2.158796293s
|
91 | 2022-08-17T20:58:11.378Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 2.628166166s
|
92 | 2022-08-17T20:58:11.859Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
93 | 2022-08-17T20:58:11.859Z | WARN | SledAgent: failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 1.427510254s
|
94 | 2022-08-17T20:58:11.900Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 2.336974445s
|
95 | 2022-08-17T20:58:11.923Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:1::1 exists
|
96 | 2022-08-17T20:58:11.923Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:1::1, prefix: 64 }))
zone = oxz_internal-dns
|
97 | 2022-08-17T20:58:12.467Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:1::1 exists - OK
|
98 | 2022-08-17T20:58:12.467Z | INFO | SledAgent (ServiceManager): GZ addresses: [
fd00:1122:3344:1::2,
]
|
99 | 2022-08-17T20:58:12.467Z | INFO | SledAgent (ServiceManager): Ensuring GZ address fd00:1122:3344:1::2 exists
|
100 | 2022-08-17T20:58:13.152Z | INFO | SledAgent (ServiceManager): Setting up internal-dns service
|
101 | 2022-08-17T20:58:13.255Z | INFO | SledAgent (ServiceManager): Sending prefix to ddmd for advertisement
prefix = Ipv6Prefix { addr: fd00:1122:3344:1::, mask: 64 }
|
102 | 2022-08-17T20:58:13.255Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:45829
req_id = 6e2af4e9-1b46-47ce-8090-df3c43c87b64
response_code = 204
uri = /services
|
103 | 2022-08-17T20:58:13.257Z | INFO | SledAgent (RSS): Adding DNS server: [fd00:1122:3344:1::1]:5353
client = DNS
|
104 | 2022-08-17T20:58:13.266Z | INFO | SledAgent (RSS): sending dataset requests...
|
105 | 2022-08-17T20:58:13.266Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 09a9a25f-2602-4e2f-9630-31af9c492c3e, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::6]:32345 }
|
106 | 2022-08-17T20:58:13.266Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:35538
|
107 | 2022-08-17T20:58:13.266Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::6]:32345, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
108 | 2022-08-17T20:58:13.266Z | INFO | SledAgent (StorageManager): Ensuring dataset oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible exists
|
109 | 2022-08-17T20:58:13.307Z | INFO | SledAgent (StorageManager): Ensuring zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible is running
|
110 | 2022-08-17T20:58:13.314Z | INFO | SledAgent (StorageManager): Zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible was not found
|
111 | 2022-08-17T20:58:13.328Z | INFO | SledAgent (StorageManager): Configuring new Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
112 | 2022-08-17T20:58:13.345Z | INFO | SledAgent (StorageManager): Installing Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
113 | 2022-08-17T20:58:14.452Z | INFO | SledAgent (StorageManager): Zone booting
zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
114 | 2022-08-17T20:58:14.568Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
115 | 2022-08-17T20:58:14.570Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.234413574s
|
116 | 2022-08-17T20:58:15.806Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
117 | 2022-08-17T20:58:15.806Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.088187175s
|
118 | 2022-08-17T20:58:16.585Z | INFO | SledAgent (StorageManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::6, prefix: 64 }))
zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
119 | 2022-08-17T20:58:17.153Z | INFO | SledAgent (StorageManager): Initializing Crucible
|
120 | 2022-08-17T20:58:17.610Z | INFO | SledAgent (StorageManager): Zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b with address [fd00:1122:3344:101::6]:32345 is running
|
121 | 2022-08-17T20:58:17.610Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
122 | 2022-08-17T20:58:17.611Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:35538
req_id = ef1cd930-fbd9-4b2b-83ef-c0830878dbf4
response_code = 204
uri = /filesystem
|
123 | 2022-08-17T20:58:17.611Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 2713b37a-3043-4ed5-aaff-f38200e45cfb, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::7]:32345 }
|
124 | 2022-08-17T20:58:17.611Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::7]:32345, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
125 | 2022-08-17T20:58:17.611Z | INFO | SledAgent (StorageManager): Ensuring dataset oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible exists
|
126 | 2022-08-17T20:58:17.613Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 530.822844ms
|
127 | 2022-08-17T20:58:17.652Z | INFO | SledAgent (StorageManager): Ensuring zone for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible is running
|
128 | 2022-08-17T20:58:17.660Z | INFO | SledAgent (StorageManager): Zone for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible was not found
|
129 | 2022-08-17T20:58:17.673Z | INFO | SledAgent (StorageManager): Configuring new Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
130 | 2022-08-17T20:58:17.690Z | INFO | SledAgent (StorageManager): Installing Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
131 | 2022-08-17T20:58:18.146Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
132 | 2022-08-17T20:58:18.147Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.151460123s
|
133 | 2022-08-17T20:58:18.796Z | INFO | SledAgent (StorageManager): Zone booting
zone = oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
134 | 2022-08-17T20:58:19.300Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
135 | 2022-08-17T20:58:19.300Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 779.036193ms
|
136 | 2022-08-17T20:58:20.080Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
137 | 2022-08-17T20:58:20.081Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 826.247023ms
|
138 | 2022-08-17T20:58:20.908Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
139 | 2022-08-17T20:58:20.909Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 574.788598ms
|
140 | 2022-08-17T20:58:21.033Z | INFO | SledAgent (StorageManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::7, prefix: 64 }))
zone = oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
141 | 2022-08-17T20:58:21.600Z | INFO | SledAgent (StorageManager): Initializing Crucible
|
142 | 2022-08-17T20:58:22.064Z | INFO | SledAgent (StorageManager): Zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 with address [fd00:1122:3344:101::7]:32345 is running
|
143 | 2022-08-17T20:58:22.064Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
144 | 2022-08-17T20:58:22.065Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:35538
req_id = e071a26d-2de1-4373-8109-f484049fe739
response_code = 204
uri = /filesystem
|
145 | 2022-08-17T20:58:22.065Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 4.314968052s
|
146 | 2022-08-17T20:58:22.065Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: ffd16cad-e5d5-495e-9c59-4312a3857d91, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
147 | 2022-08-17T20:58:22.065Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 3.317223484s
|
148 | 2022-08-17T20:58:22.065Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
149 | 2022-08-17T20:58:22.065Z | INFO | SledAgent (StorageManager): Ensuring dataset oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible exists
|
150 | 2022-08-17T20:58:22.068Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 873.171137ms
|
151 | 2022-08-17T20:58:22.106Z | INFO | SledAgent (StorageManager): Ensuring zone for oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible is running
|
152 | 2022-08-17T20:58:22.113Z | INFO | SledAgent (StorageManager): Zone for oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible was not found
|
153 | 2022-08-17T20:58:22.125Z | INFO | SledAgent (StorageManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
154 | 2022-08-17T20:58:22.141Z | INFO | SledAgent (StorageManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
155 | 2022-08-17T20:58:22.942Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
156 | 2022-08-17T20:58:22.943Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.176810436s
|
157 | 2022-08-17T20:58:23.263Z | INFO | SledAgent (StorageManager): Zone booting
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
158 | 2022-08-17T20:58:24.122Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
159 | 2022-08-17T20:58:24.123Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 546.071049ms
|
160 | 2022-08-17T20:58:24.669Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
161 | 2022-08-17T20:58:24.670Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 944.545224ms
|
162 | 2022-08-17T20:58:25.055Z | INFO | SledAgent (StorageManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::8, prefix: 64 }))
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
163 | 2022-08-17T20:58:25.623Z | INFO | SledAgent (StorageManager): Initializing Crucible
|
164 | 2022-08-17T20:58:26.096Z | INFO | SledAgent (StorageManager): Zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 with address [fd00:1122:3344:101::8]:32345 is running
|
165 | 2022-08-17T20:58:26.097Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
166 | 2022-08-17T20:58:26.097Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:35538
req_id = 9eeeb205-d3f4-410c-8ae1-a85a347341e4
response_code = 204
uri = /filesystem
|
167 | 2022-08-17T20:58:26.097Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 5.044761756s
|
168 | 2022-08-17T20:58:26.097Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 4d08fc19-3d5f-4f6b-9c48-925f8eac7255, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: CockroachDb { all_addresses: [[fd00:1122:3344:101::2]:32221] }, address: [fd00:1122:3344:101::2]:32221 }
|
169 | 2022-08-17T20:58:26.097Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 5.797654437s
|
170 | 2022-08-17T20:58:26.097Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 127.89601ms
|
171 | 2022-08-17T20:58:26.097Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: CockroachDb { all_addresses: [[fd00:1122:3344:101::2]:32221] }, address: [fd00:1122:3344:101::2]:32221, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
172 | 2022-08-17T20:58:26.097Z | INFO | SledAgent (StorageManager): Ensuring dataset oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/cockroachdb exists
|
173 | 2022-08-17T20:58:26.097Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 919.546203ms
|
174 | 2022-08-17T20:58:26.140Z | INFO | SledAgent (StorageManager): Ensuring zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/cockroachdb is running
|
175 | 2022-08-17T20:58:26.148Z | INFO | SledAgent (StorageManager): Zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/cockroachdb was not found
|
176 | 2022-08-17T20:58:26.162Z | INFO | SledAgent (StorageManager): Configuring new Omicron zone: oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
177 | 2022-08-17T20:58:26.179Z | INFO | SledAgent (StorageManager): Installing Omicron zone: oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
178 | 2022-08-17T20:58:27.019Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
179 | 2022-08-17T20:58:27.020Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.490968136s
|
180 | 2022-08-17T20:58:28.265Z | INFO | SledAgent (StorageManager): Zone booting
zone = oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
181 | 2022-08-17T20:58:28.512Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
182 | 2022-08-17T20:58:28.513Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 859.868621ms
|
183 | 2022-08-17T20:58:29.374Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
184 | 2022-08-17T20:58:29.374Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 989.003288ms
|
185 | 2022-08-17T20:58:30.365Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
186 | 2022-08-17T20:58:30.365Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 730.310118ms
|
187 | 2022-08-17T20:58:30.840Z | INFO | SledAgent (StorageManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))
zone = oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
188 | 2022-08-17T20:58:31.407Z | INFO | SledAgent (StorageManager): start_zone: Loading CRDB manifest
|
189 | 2022-08-17T20:58:31.600Z | INFO | SledAgent (StorageManager): start_zone: setting CRDB's config/listen_addr: [fd00:1122:3344:101::2]:32221
|
190 | 2022-08-17T20:58:31.628Z | INFO | SledAgent (StorageManager): start_zone: setting CRDB's config/store
|
191 | 2022-08-17T20:58:31.657Z | INFO | SledAgent (StorageManager): start_zone: setting CRDB's config/join_addrs
|
192 | 2022-08-17T20:58:31.684Z | INFO | SledAgent (StorageManager): start_zone: refreshing manifest
|
193 | 2022-08-17T20:58:31.711Z | INFO | SledAgent (StorageManager): start_zone: enabling CRDB service
|
194 | 2022-08-17T20:58:31.737Z | INFO | SledAgent (StorageManager): start_zone: awaiting liveness of CRDB
|
195 | 2022-08-17T20:58:31.743Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
196 | 2022-08-17T20:58:31.743Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.476221319s
|
197 | 2022-08-17T20:58:31.744Z | WARN | SledAgent (StorageManager): cockroachdb not yet alive
|
198 | 2022-08-17T20:58:31.996Z | WARN | SledAgent (StorageManager): cockroachdb not yet alive
|
199 | 2022-08-17T20:58:32.401Z | WARN | SledAgent (StorageManager): cockroachdb not yet alive
|
200 | 2022-08-17T20:58:33.220Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
201 | 2022-08-17T20:58:33.221Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 923.543475ms
|
202 | 2022-08-17T20:58:33.725Z | WARN | SledAgent (StorageManager): cockroachdb not yet alive
|
203 | 2022-08-17T20:58:34.146Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
204 | 2022-08-17T20:58:34.147Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 714.773697ms
|
205 | 2022-08-17T20:58:34.863Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
206 | 2022-08-17T20:58:34.864Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 936.986186ms
|
207 | 2022-08-17T20:58:34.971Z | INFO | SledAgent (StorageManager): CRDB is online
|
208 | 2022-08-17T20:58:34.971Z | INFO | SledAgent (StorageManager): Formatting CRDB
|
209 | 2022-08-17T20:58:35.801Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
210 | 2022-08-17T20:58:35.802Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.083216901s
|
211 | 2022-08-17T20:58:36.886Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
212 | 2022-08-17T20:58:36.887Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.10148591s
|
213 | 2022-08-17T20:58:37.989Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
214 | 2022-08-17T20:58:37.990Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.164952307s
|
215 | 2022-08-17T20:58:39.155Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
216 | 2022-08-17T20:58:39.156Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.101061878s
|
217 | 2022-08-17T20:58:40.258Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
218 | 2022-08-17T20:58:40.259Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.372327978s
|
219 | 2022-08-17T20:58:41.632Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
220 | 2022-08-17T20:58:41.632Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 588.979295ms
|
221 | 2022-08-17T20:58:42.222Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
222 | 2022-08-17T20:58:42.223Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 598.128413ms
|
223 | 2022-08-17T20:58:42.339Z | INFO | SledAgent (StorageManager): Formatting CRDB - Completed
|
224 | 2022-08-17T20:58:42.339Z | INFO | SledAgent (StorageManager): Zone oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b with address [fd00:1122:3344:101::2]:32221 is running
|
225 | 2022-08-17T20:58:42.339Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:35538
req_id = c4085914-8e51-430f-94ed-08d75f8c1014
response_code = 204
uri = /filesystem
|
226 | 2022-08-17T20:58:42.339Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: a3505b41-a592-420b-84f2-3d76bf0e0a81, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Clickhouse, address: [fd00:1122:3344:101::5]:8123 }
|
227 | 2022-08-17T20:58:42.339Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Clickhouse, address: [fd00:1122:3344:101::5]:8123, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
228 | 2022-08-17T20:58:42.339Z | INFO | SledAgent (StorageManager): Ensuring dataset oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/clickhouse exists
|
229 | 2022-08-17T20:58:42.388Z | INFO | SledAgent (StorageManager): Ensuring zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/clickhouse is running
|
230 | 2022-08-17T20:58:42.398Z | INFO | SledAgent (StorageManager): Zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/clickhouse was not found
|
231 | 2022-08-17T20:58:42.414Z | INFO | SledAgent (StorageManager): Configuring new Omicron zone: oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
232 | 2022-08-17T20:58:42.431Z | INFO | SledAgent (StorageManager): Installing Omicron zone: oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
233 | 2022-08-17T20:58:42.823Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
234 | 2022-08-17T20:58:42.824Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 620.343174ms
|
235 | 2022-08-17T20:58:43.446Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
236 | 2022-08-17T20:58:43.446Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 688.509497ms
|
237 | 2022-08-17T20:58:44.137Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
238 | 2022-08-17T20:58:44.137Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.476753429s
|
239 | 2022-08-17T20:58:44.623Z | INFO | SledAgent (StorageManager): Zone booting
zone = oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
240 | 2022-08-17T20:58:45.616Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
241 | 2022-08-17T20:58:45.617Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 786.016957ms
|
242 | 2022-08-17T20:58:46.250Z | INFO | SledAgent (StorageManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::5, prefix: 64 }))
zone = oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
243 | 2022-08-17T20:58:46.822Z | INFO | SledAgent (StorageManager): Initialiting Clickhouse
|
244 | 2022-08-17T20:58:47.120Z | INFO | SledAgent (StorageManager): Zone oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b with address [fd00:1122:3344:101::5]:8123 is running
|
245 | 2022-08-17T20:58:47.120Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
246 | 2022-08-17T20:58:47.121Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:35538
req_id = b03e4d17-47ff-43f1-b28c-7b6d73962ec4
response_code = 204
uri = /filesystem
|
247 | 2022-08-17T20:58:47.121Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 901.036969ms
|
248 | 2022-08-17T20:58:47.121Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 5.360915826s
|
249 | 2022-08-17T20:58:47.122Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 189.422432ms
|
250 | 2022-08-17T20:58:47.122Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 4.431716735s
|
251 | 2022-08-17T20:58:47.123Z | INFO | SledAgent (RSS): Inserting DNS record: Backend(Crucible, ffd16cad-e5d5-495e-9c59-4312a3857d91)
client = DNS
|
252 | 2022-08-17T20:58:47.123Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 264.749397ms
|
253 | 2022-08-17T20:58:47.123Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 417.502891ms
|
254 | 2022-08-17T20:58:47.123Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 193.329646ms
|
255 | 2022-08-17T20:58:47.124Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 8.596784115s
|
256 | 2022-08-17T20:58:47.124Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.502197423s
|
257 | 2022-08-17T20:58:47.124Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 6.596661124s
|
258 | 2022-08-17T20:58:47.125Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 6.416293599s
|
259 | 2022-08-17T20:58:47.125Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 288.52532ms
|
260 | 2022-08-17T20:58:47.126Z | INFO | SledAgent (RSS): Inserting DNS record: Service(Clickhouse)
client = DNS
|
261 | 2022-08-17T20:58:47.126Z | INFO | SledAgent (RSS): Inserting DNS record: Service(Cockroach)
client = DNS
|
262 | 2022-08-17T20:58:47.126Z | INFO | SledAgent (RSS): Inserting DNS record: Backend(Crucible, 09a9a25f-2602-4e2f-9630-31af9c492c3e)
client = DNS
|
263 | 2022-08-17T20:58:47.127Z | INFO | SledAgent (RSS): Inserting DNS record: Backend(Crucible, 2713b37a-3043-4ed5-aaff-f38200e45cfb)
client = DNS
|
264 | 2022-08-17T20:58:47.127Z | INFO | SledAgent (RSS): Finished setting up agents and datasets
|
265 | 2022-08-17T20:58:47.131Z | INFO | SledAgent (RSS): sending service requests...
|
266 | 2022-08-17T20:58:47.131Z | INFO | SledAgent (RSS): initializing sled services: [ServiceRequest { id: e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c, name: "nexus", addresses: [fd00:1122:3344:101::3], gz_addresses: [], service_type: Nexus { internal_ip: fd00:1122:3344:101::3, external_ip: 192.168.1.20 } }, ServiceRequest { id: 1da65e5b-210c-4859-a7d7-200c1e659972, name: "oximeter", addresses: [fd00:1122:3344:101::4], gz_addresses: [], service_type: Oximeter }, ServiceRequest { id: a0fe5ebc-9261-6f77-acc1-972481755789, name: "dendrite", addresses: [fd00:1122:3344:101::9], gz_addresses: [], service_type: Dendrite { asic: TofinoStub } }, ServiceRequest { id: 4b51d2cc-ccf1-49ea-b3bb-0463bb1cff8b, name: "internal-dns", addresses: [fd00:1122:3344:1::1], gz_addresses: [fd00:1122:3344:1::2], service_type: InternalDns { server_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53 } }]
|
267 | 2022-08-17T20:58:47.131Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:61828
|
268 | 2022-08-17T20:58:47.131Z | INFO | SledAgent (ServiceManager): Ensuring service is initialized: ServiceRequest { id: 1da65e5b-210c-4859-a7d7-200c1e659972, name: "oximeter", addresses: [fd00:1122:3344:101::4], gz_addresses: [], service_type: Oximeter }
|
269 | 2022-08-17T20:58:47.132Z | INFO | SledAgent (ServiceManager): Service oximeter does not yet exist
|
270 | 2022-08-17T20:58:47.148Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_oximeter
|
271 | 2022-08-17T20:58:47.166Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_oximeter
|
272 | 2022-08-17T20:58:47.314Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 452.869246ms
|
273 | 2022-08-17T20:58:47.318Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 521.146104ms
|
274 | 2022-08-17T20:58:47.389Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 250.175528ms
|
275 | 2022-08-17T20:58:47.415Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 582.144102ms
|
276 | 2022-08-17T20:58:47.542Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 800.472421ms
|
277 | 2022-08-17T20:58:47.640Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.393655769s
|
278 | 2022-08-17T20:58:47.768Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 916.460152ms
|
279 | 2022-08-17T20:58:47.840Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 832.022872ms
|
280 | 2022-08-17T20:58:47.989Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_oximeter
|
281 | 2022-08-17T20:58:47.999Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 876.593613ms
|
282 | 2022-08-17T20:58:48.023Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
283 | 2022-08-17T20:58:48.024Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 596.074706ms
|
284 | 2022-08-17T20:58:48.345Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.719146054s
|
285 | 2022-08-17T20:58:48.621Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
286 | 2022-08-17T20:58:48.622Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 599.622175ms
|
287 | 2022-08-17T20:58:48.674Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.762387676s
|
288 | 2022-08-17T20:58:48.686Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.905289499s
|
289 | 2022-08-17T20:58:48.878Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.480041144s
|
290 | 2022-08-17T20:58:49.036Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.660389238s
|
291 | 2022-08-17T20:58:49.222Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
292 | 2022-08-17T20:58:49.223Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.484687811s
|
293 | 2022-08-17T20:58:49.628Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 10.769379266s
|
294 | 2022-08-17T20:58:49.745Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::4 exists
|
295 | 2022-08-17T20:58:49.745Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::4, prefix: 64 }))
zone = oxz_oximeter
|
296 | 2022-08-17T20:58:50.296Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::4 exists - OK
|
297 | 2022-08-17T20:58:50.296Z | INFO | SledAgent (ServiceManager): GZ addresses: []
|
298 | 2022-08-17T20:58:50.511Z | INFO | SledAgent (ServiceManager): Setting up oximeter service
|
299 | 2022-08-17T20:58:50.613Z | INFO | SledAgent (ServiceManager): Ensuring service is initialized: ServiceRequest { id: e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c, name: "nexus", addresses: [fd00:1122:3344:101::3], gz_addresses: [], service_type: Nexus { internal_ip: fd00:1122:3344:101::3, external_ip: 192.168.1.20 } }
|
300 | 2022-08-17T20:58:50.613Z | INFO | SledAgent (ServiceManager): Service nexus does not yet exist
|
301 | 2022-08-17T20:58:50.633Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus
|
302 | 2022-08-17T20:58:50.649Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus
|
303 | 2022-08-17T20:58:51.702Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_nexus
|
304 | 2022-08-17T20:58:51.880Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
305 | 2022-08-17T20:58:51.881Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.208346905s
|
306 | 2022-08-17T20:58:51.881Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.380203515s
|
307 | 2022-08-17T20:58:51.881Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 4.421353536s
|
308 | 2022-08-17T20:58:51.882Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 5.772832278s
|
309 | 2022-08-17T20:58:51.882Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.612661851s
|
310 | 2022-08-17T20:58:51.882Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.71701267s
|
311 | 2022-08-17T20:58:51.883Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 17.243687282s
|
312 | 2022-08-17T20:58:52.484Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 10.635449452s
|
313 | 2022-08-17T20:58:53.090Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
314 | 2022-08-17T20:58:53.091Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.154766346s
|
315 | 2022-08-17T20:58:53.544Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 13.432634823s
|
316 | 2022-08-17T20:58:53.724Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 14.471515206s
|
317 | 2022-08-17T20:58:54.105Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::3 exists
|
318 | 2022-08-17T20:58:54.105Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::3, prefix: 64 }))
zone = oxz_nexus
|
319 | 2022-08-17T20:58:54.655Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::3 exists - OK
|
320 | 2022-08-17T20:58:54.655Z | INFO | SledAgent (ServiceManager): GZ addresses: []
|
321 | 2022-08-17T20:58:54.876Z | INFO | SledAgent (ServiceManager): Setting up Nexus service
|
322 | 2022-08-17T20:58:54.877Z | INFO | SledAgent (ServiceManager): Adding address: Static(V4(Ipv4Network { addr: 192.168.1.20, prefix: 24 }))
zone = oxz_nexus
|
323 | 2022-08-17T20:58:55.380Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
324 | 2022-08-17T20:58:55.384Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 612.703445ms
|
325 | 2022-08-17T20:58:55.384Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 5.444035499s
|
326 | 2022-08-17T20:58:55.384Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 6.649068817s
|
327 | 2022-08-17T20:58:55.385Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 6.632216383s
|
328 | 2022-08-17T20:58:55.407Z | INFO | SledAgent (ServiceManager): Ensuring service is initialized: ServiceRequest { id: a0fe5ebc-9261-6f77-acc1-972481755789, name: "dendrite", addresses: [fd00:1122:3344:101::9], gz_addresses: [], service_type: Dendrite { asic: TofinoStub } }
|
329 | 2022-08-17T20:58:55.407Z | INFO | SledAgent (ServiceManager): Service dendrite does not yet exist
|
330 | 2022-08-17T20:58:55.424Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_dendrite
|
331 | 2022-08-17T20:58:55.440Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_dendrite
|
332 | 2022-08-17T20:58:55.723Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 41.573582228s
|
333 | 2022-08-17T20:58:55.998Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
334 | 2022-08-17T20:58:55.998Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 815.22759ms
|
335 | 2022-08-17T20:58:56.304Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 7.492087105s
|
336 | 2022-08-17T20:58:56.815Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
337 | 2022-08-17T20:58:56.846Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.013406456s
|
338 | 2022-08-17T20:58:57.656Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 4.642613057s
|
339 | 2022-08-17T20:58:57.860Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
340 | 2022-08-17T20:58:57.861Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 502.578922ms
|
341 | 2022-08-17T20:58:57.917Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_dendrite
|
342 | 2022-08-17T20:58:58.365Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
343 | 2022-08-17T20:58:58.366Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.418853804s
|
344 | 2022-08-17T20:58:59.785Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
345 | 2022-08-17T20:58:59.786Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.403741363s
|
346 | 2022-08-17T20:59:00.399Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 20.403475875s
|
347 | 2022-08-17T20:59:00.537Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::9 exists
|
348 | 2022-08-17T20:59:00.537Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::9, prefix: 64 }))
zone = oxz_dendrite
|
349 | 2022-08-17T20:59:01.084Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::9 exists - OK
|
350 | 2022-08-17T20:59:01.085Z | INFO | SledAgent (ServiceManager): GZ addresses: []
|
351 | 2022-08-17T20:59:01.298Z | INFO | SledAgent (ServiceManager): Setting up dendrite service
|
352 | 2022-08-17T20:59:01.406Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
353 | 2022-08-17T20:59:01.406Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61828
req_id = 057181ff-f185-48f3-b9d9-1b4faa0e4b1d
response_code = 204
uri = /services
|
354 | 2022-08-17T20:59:01.407Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.354918048s
|
355 | 2022-08-17T20:59:01.407Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 9.08685705s
|
356 | 2022-08-17T20:59:01.408Z | INFO | SledAgent (RSS): Inserting DNS record: Service(Nexus)
client = DNS
|
357 | 2022-08-17T20:59:01.408Z | INFO | SledAgent (RSS): Inserting DNS record: Service(Oximeter)
client = DNS
|
358 | 2022-08-17T20:59:01.408Z | INFO | SledAgent (RSS): Inserting DNS record: Service(Dendrite)
client = DNS
|
359 | 2022-08-17T20:59:01.409Z | INFO | SledAgent (RSS): Inserting DNS record: Service(InternalDNS)
client = DNS
|
360 | 2022-08-17T20:59:01.409Z | INFO | SledAgent (RSS): Finished setting up services
|
361 | 2022-08-17T20:59:02.024Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 21.145248043s
|
362 | 2022-08-17T20:59:02.040Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 16.720685243s
|
363 | 2022-08-17T20:59:02.305Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 17.869728932s
|
364 | 2022-08-17T20:59:02.763Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
365 | 2022-08-17T20:59:02.768Z | WARN | SledAgent: failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::3]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 1.097317357s
|
366 | 2022-08-17T20:59:03.802Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 16.271864098s
|
367 | 2022-08-17T20:59:03.867Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
368 | 2022-08-17T20:59:03.872Z | WARN | SledAgent: failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::3]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 1.007270171s
|
369 | 2022-08-17T20:59:04.880Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
370 | 2022-08-17T20:59:04.885Z | WARN | SledAgent: failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::3]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 907.079329ms
|
371 | 2022-08-17T20:59:05.794Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
372 | 2022-08-17T20:59:05.799Z | WARN | SledAgent: failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::3]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 1.248181063s
|
373 | 2022-08-17T20:59:06.990Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 22.890549373s
|
374 | 2022-08-17T20:59:07.048Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
|