| 1 | | [ Nov 28 20:29:53 Enabled. ] |
| 2 | | [ Nov 28 20:29:53 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] |
| 3 | | [ Nov 28 20:29:53 Method "start" exited with status 0. ] |
| 4 | | [ Nov 28 20:29:53 Rereading configuration. ] |
| 5 | | [ Nov 28 20:29:53 No 'refresh' method defined. Treating as :true. ] |
| 6 | | note: configured to log to "/dev/stdout" |
| 7 | 2022-11-28T20:29:53.445Z | INFO | SledAgent: Starting mg-ddm service
|
| 8 | 2022-11-28T20:29:53.445Z | INFO | SledAgent: Importing mg-ddm service
path = /opt/oxide/mg-ddm/pkg/ddm/manifest.xml
|
| 9 | 2022-11-28T20:29:53.556Z | INFO | SledAgent: Setting mg-ddm interfaces
interfaces = ("net0/linklocal" "net1/linklocal")
|
| 10 | 2022-11-28T20:29:53.564Z | INFO | SledAgent: Enabling mg-ddm service
|
| 11 | 2022-11-28T20:29:53.572Z | INFO | SledAgent: detecting (real or simulated) SP
|
| 12 | 2022-11-28T20:29:53.572Z | INFO | SledAgent: setting up bootstrap agent server
|
| 13 | 2022-11-28T20:29:53.572Z | INFO | SledAgent: Ensuring config directory exists
path = /var/oxide
|
| 14 | 2022-11-28T20:29:54.074Z | INFO | SledAgent (BootstrapAgentServer): Started listening
local_addr = [fdb0:18c0:4d0d:9fb2::1]:12346
|
| 15 | 2022-11-28T20:29:54.074Z | INFO | SledAgent: Sending prefix to ddmd for advertisement
prefix = Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, mask: 64 }
|
| 16 | 2022-11-28T20:29:54.074Z | INFO | SledAgent (BootstrapAgent): bootstrap service initializing RSS
server = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 17 | 2022-11-28T20:29:54.075Z | INFO | SledAgent (RSS): Injecting RSS configuration: SetupServiceConfig {
rack_subnet: fd00:1122:3344:100::,
requests: [
HardcodedSledRequest {
datasets: [
DatasetEnsureBody {
id: 09a9a25f26024e2f963031af9c492c3e,
zpool_id: d462a7f7b62840fe80ff4e4189e2d62b,
dataset_kind: Crucible,
address: [fd00:1122:3344:101::7]:32345,
},
DatasetEnsureBody {
id: 2713b37a30434ed5aafff38200e45cfb,
zpool_id: e4b4dc87ab4649fba4b4d361ae214c03,
dataset_kind: Crucible,
address: [fd00:1122:3344:101::8]:32345,
},
DatasetEnsureBody {
id: ffd16cade5d5495e9c594312a3857d91,
zpool_id: f4b4dc87ab4649fba4b4d361ae214c03,
dataset_kind: Crucible,
address: [fd00:1122:3344:101::9]:32345,
},
DatasetEnsureBody {
id: 4d08fc193d5f4f6b9c48925f8eac7255,
zpool_id: d462a7f7b62840fe80ff4e4189e2d62b,
dataset_kind: CockroachDb {
all_addresses: [
[fd00:1122:3344:101::3]:32221,
],
},
address: [fd00:1122:3344:101::3]:32221,
},
DatasetEnsureBody {
id: a3505b41a592420b84f23d76bf0e0a81,
zpool_id: d462a7f7b62840fe80ff4e4189e2d62b,
dataset_kind: Clickhouse,
address: [fd00:1122:3344:101::6]:8123,
},
],
service_zones: [
ServiceZoneRequest {
id: e6bff1ff24fb49dca54ec6a350cd4d6c,
zone_type: Nexus,
addresses: [
fd00:1122:3344:101::4,
],
gz_addresses: [],
services: [
Nexus {
internal_ip: fd00:1122:3344:101::4,
external_ip: 192.168.1.20,
},
],
},
ServiceZoneRequest {
id: 1da65e5b210c4859a7d7200c1e659972,
zone_type: Oximeter,
addresses: [
fd00:1122:3344:101::5,
],
gz_addresses: [],
services: [
Oximeter,
],
},
],
dns_services: [],
},
],
rack_secret_threshold: 1,
gateway: Gateway {
address: Some(
192.168.1.199,
),
mac: MacAddr6(
[
24,
192,
77,
13,
159,
178,
],
),
},
}
|
| 18 | 2022-11-28T20:29:54.075Z | INFO | SledAgent (RSS): RSS configuration has not been fully applied yet
|
| 19 | 2022-11-28T20:29:54.080Z | INFO | SledAgent (RSS): Received prefixes from ddmd
prefixes = {"fe80::8:20ff:fe84:a595": [Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, mask: 64 }], "fe80::8:20ff:fe1f:d856": [Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, mask: 64 }]}
|
| 20 | 2022-11-28T20:29:54.081Z | INFO | SledAgent (RSS): Enough peers exist to enact RSS plan
|
| 21 | 2022-11-28T20:29:54.081Z | INFO | SledAgent (RSS): Creating new allocation plan
|
| 22 | 2022-11-28T20:29:54.081Z | INFO | SledAgent (RSS): dns_subnets: [
DnsSubnet {
subnet: Ipv6Subnet {
net: Ipv6Net(
Ipv6Network {
addr: fd00:1122:3344:1::,
prefix: 64,
},
),
},
},
]
|
| 23 | 2022-11-28T20:29:54.081Z | INFO | SledAgent (RSS): Serializing plan
|
| 24 | 2022-11-28T20:29:54.081Z | INFO | SledAgent (RSS): Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1
|
| 25 | 2022-11-28T20:29:54.081Z | INFO | SledAgent (RSS): Plan serialized as: ["[fdb0:18c0:4d0d:9fb2::1]:12346".initialization_request]
id = "3b68b78f-7c48-48aa-a6f8-7b0b244f483f"
rack_id = "13b5eda3-d280-4d3d-92d5-3e0b107addcc"
["[fdb0:18c0:4d0d:9fb2::1]:12346".initialization_request.gateway]
address = "192.168.1.199"
mac = [24, 192, 77, 13, 159, 178]
["[fdb0:18c0:4d0d:9fb2::1]:12346".initialization_request.subnet]
net = "fd00:1122:3344:101::/64"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset]]
address = "[fd00:1122:3344:101::7]:32345"
id = "09a9a25f-2602-4e2f-9630-31af9c492c3e"
zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b"
["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset.dataset_kind]
type = "crucible"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset]]
address = "[fd00:1122:3344:101::8]:32345"
id = "2713b37a-3043-4ed5-aaff-f38200e45cfb"
zpool_id = "e4b4dc87-ab46-49fb-a4b4-d361ae214c03"
["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset.dataset_kind]
type = "crucible"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset]]
address = "[fd00:1122:3344:101::9]:32345"
id = "ffd16cad-e5d5-495e-9c59-4312a3857d91"
zpool_id = "f4b4dc87-ab46-49fb-a4b4-d361ae214c03"
["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset.dataset_kind]
type = "crucible"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset]]
address = "[fd00:1122:3344:101::3]:32221"
id = "4d08fc19-3d5f-4f6b-9c48-925f8eac7255"
zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b"
["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset.dataset_kind]
all_addresses = ["[fd00:1122:3344:101::3]:32221"]
type = "cockroach_db"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset]]
address = "[fd00:1122:3344:101::6]:8123"
id = "a3505b41-a592-420b-84f2-3d76bf0e0a81"
zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b"
["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset.dataset_kind]
type = "clickhouse"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dns_service]]
addresses = ["fd00:1122:3344:1::1"]
gz_addresses = ["fd00:1122:3344:1::2"]
id = "b8568fdf-8b3e-40d6-ae1a-8737e10bc1dc"
zone_type = "internal_dns"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dns_service.services]]
dns_address = "[fd00:1122:3344:1::1]:53"
server_address = "[fd00:1122:3344:1::1]:5353"
type = "internal_dns"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.service_zone]]
addresses = ["fd00:1122:3344:101::4"]
gz_addresses = []
id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"
zone_type = "nexus"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.service_zone.services]]
external_ip = "192.168.1.20"
internal_ip = "fd00:1122:3344:101::4"
type = "nexus"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.service_zone]]
addresses = ["fd00:1122:3344:101::5"]
gz_addresses = []
id = "1da65e5b-210c-4859-a7d7-200c1e659972"
zone_type = "oximeter"
[["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.service_zone.services]]
type = "oximeter"
|
| 26 | 2022-11-28T20:29:54.081Z | INFO | SledAgent (RSS): Plan written to storage
|
| 27 | 2022-11-28T20:29:54.081Z | INFO | SledAgent (RSS): Skipping rack secret creation (only one sled present)
|
| 28 | 2022-11-28T20:29:54.081Z | INFO | SledAgent (BootstrapAgentRssHandler): Received initialization request from RSS
request = SledAgentRequest { id: 3b68b78f-7c48-48aa-a6f8-7b0b244f483f, rack_id: 13b5eda3-d280-4d3d-92d5-3e0b107addcc, gateway: Gateway { address: Some(192.168.1.199), mac: MacAddr6([24, 192, 77, 13, 159, 178]) }, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } }
target_sled = [fdb0:18c0:4d0d:9fb2::1]:12346
|
| 29 | 2022-11-28T20:29:54.081Z | INFO | SledAgent (BootstrapAgentServer): Accepted connection
remote_addr = [fdb0:18c0:4d0d:9fb2::1]:49379
|
| 30 | 2022-11-28T20:29:54.081Z | INFO | SledAgent (BootstrapAgentRssHandler): No SP available; proceeding without sprockets auth
BootstrapAgentClient = [fdb0:18c0:4d0d:9fb2::1]:12346
|
| 31 | 2022-11-28T20:29:54.081Z | INFO | SledAgent (BootstrapAgentServer): No SP available; proceeding without sprockets auth
remote_addr = [fdb0:18c0:4d0d:9fb2::1]:49379
|
| 32 | 2022-11-28T20:29:54.083Z | INFO | SledAgent (BootstrapAgent): Loading Sled Agent: SledAgentRequest { id: 3b68b78f-7c48-48aa-a6f8-7b0b244f483f, rack_id: 13b5eda3-d280-4d3d-92d5-3e0b107addcc, gateway: Gateway { address: Some(192.168.1.199), mac: MacAddr6([24, 192, 77, 13, 159, 178]) }, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } }
server = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 33 | 2022-11-28T20:29:54.084Z | INFO | SledAgent: setting up sled agent server
|
| 34 | 2022-11-28T20:29:54.084Z | INFO | SledAgent: created sled agent
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 35 | 2022-11-28T20:29:54.575Z | INFO | SledAgent: updating xde driver configuration file for external IP workaround
conf_file = "/kernel/drv/xde.conf"
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 36 | 2022-11-28T20:29:54.621Z | INFO | SledAgent: using '[AddrObject { interface: "net0", name: "linklocal" }, AddrObject { interface: "net1", name: "linklocal" }]' as data links for xde driver
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 37 | 2022-11-28T20:29:54.754Z | INFO | SledAgent: Sled Agent upserting zpool to Storage Manager: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 38 | 2022-11-28T20:29:54.763Z | INFO | SledAgent: Sled Agent upserting zpool to Storage Manager: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 39 | 2022-11-28T20:29:54.770Z | INFO | SledAgent: Sled Agent upserting zpool to Storage Manager: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 40 | 2022-11-28T20:29:54.782Z | INFO | SledAgent (StorageManager): Storage manager processing zpool: ZpoolInfo {
name: "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b",
size: 10200547328,
allocated: 127488,
free: 10200419840,
health: Online,
}
|
| 41 | 2022-11-28T20:29:54.791Z | INFO | SledAgent (StorageManager): Storage manager processing zpool: ZpoolInfo {
name: "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03",
size: 10200547328,
allocated: 125952,
free: 10200421376,
health: Online,
}
|
| 42 | 2022-11-28T20:29:54.798Z | INFO | SledAgent (ServiceManager): No sled services found at /var/oxide/service.toml
|
| 43 | 2022-11-28T20:29:54.798Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 44 | 2022-11-28T20:29:54.798Z | INFO | SledAgent (dropshot (SledAgent)): listening
local_addr = [fd00:1122:3344:101::1]:12345
|
| 45 | 2022-11-28T20:29:54.799Z | INFO | SledAgent (BootstrapAgent): Sled Agent loaded; recording configuration
server = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 46 | 2022-11-28T20:29:54.799Z | INFO | SledAgent (HardwareManager): Monitoring for hardware updates
|
| 47 | 2022-11-28T20:29:54.799Z | INFO | SledAgent: Performing full hardware scan
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 48 | 2022-11-28T20:29:54.799Z | INFO | SledAgent: Ensuring scrimlet services (disabling services)
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 49 | 2022-11-28T20:29:54.799Z | INFO | SledAgent (ServiceManager): Disabling switch zone (already complete)
|
| 50 | 2022-11-28T20:29:54.799Z | INFO | SledAgent: Sending prefix to ddmd for advertisement
prefix = Ipv6Prefix { addr: fd00:1122:3344:101::, mask: 64 }
|
| 51 | 2022-11-28T20:29:54.799Z | INFO | SledAgent (BootstrapAgentServer): Connection closed
remote_addr = [fdb0:18c0:4d0d:9fb2::1]:49379
|
| 52 | 2022-11-28T20:29:54.799Z | INFO | SledAgent (BootstrapAgentRssHandler): Peer agent initialized
peer = [fdb0:18c0:4d0d:9fb2::1]:12346
|
| 53 | 2022-11-28T20:29:54.799Z | INFO | SledAgent (BootstrapAgentRssHandler): Initialized sled agent
target_sled = [fdb0:18c0:4d0d:9fb2::1]:12346
|
| 54 | 2022-11-28T20:29:54.799Z | WARN | SledAgent: failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 359.445298ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 55 | 2022-11-28T20:29:54.800Z | INFO | SledAgent (StorageManager): Storage manager processing zpool: ZpoolInfo {
name: "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03",
size: 10200547328,
allocated: 125952,
free: 10200421376,
health: Online,
}
|
| 56 | 2022-11-28T20:29:54.807Z | INFO | SledAgent (RSS): sending service requests...
|
| 57 | 2022-11-28T20:29:54.807Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { id: b8568fdf-8b3e-40d6-ae1a-8737e10bc1dc, zone_type: InternalDNS, addresses: [fd00:1122:3344:1::1], gz_addresses: [fd00:1122:3344:1::2], services: [InternalDns { server_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53 }] }]
|
| 58 | 2022-11-28T20:29:54.807Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:56173
|
| 59 | 2022-11-28T20:29:54.807Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: InternalDNS
|
| 60 | 2022-11-28T20:29:54.807Z | INFO | SledAgent (ServiceManager): Service zone internal_dns does not yet exist
|
| 61 | 2022-11-28T20:29:54.811Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 335.490899ms
|
| 62 | 2022-11-28T20:29:54.811Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 196.271834ms
|
| 63 | 2022-11-28T20:29:54.811Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 266.481293ms
|
| 64 | 2022-11-28T20:29:54.811Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 275.160319ms
|
| 65 | 2022-11-28T20:29:54.811Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 297.978187ms
|
| 66 | 2022-11-28T20:29:54.812Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 270.08871ms
|
| 67 | 2022-11-28T20:29:54.820Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns
|
| 68 | 2022-11-28T20:29:54.839Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns
|
| 69 | 2022-11-28T20:29:55.009Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 683.632659ms
|
| 70 | 2022-11-28T20:29:55.079Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 544.544046ms
|
| 71 | 2022-11-28T20:29:55.083Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 692.361989ms
|
| 72 | 2022-11-28T20:29:55.089Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 618.828967ms
|
| 73 | 2022-11-28T20:29:55.111Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 313.784078ms
|
| 74 | 2022-11-28T20:29:55.148Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 539.797461ms
|
| 75 | 2022-11-28T20:29:55.160Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 76 | 2022-11-28T20:29:55.160Z | WARN | SledAgent: failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 423.268178ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 77 | 2022-11-28T20:29:55.426Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 640.051924ms
|
| 78 | 2022-11-28T20:29:55.585Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 79 | 2022-11-28T20:29:55.586Z | WARN | SledAgent: failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 910.209008ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 80 | 2022-11-28T20:29:55.625Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 903.451155ms
|
| 81 | 2022-11-28T20:29:55.689Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 941.303914ms
|
| 82 | 2022-11-28T20:29:55.694Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 1.457379223s
|
| 83 | 2022-11-28T20:29:55.709Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 774.799877ms
|
| 84 | 2022-11-28T20:29:55.777Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.243983362s
|
| 85 | 2022-11-28T20:29:56.067Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 1.172833731s
|
| 86 | 2022-11-28T20:29:56.068Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_internal_dns
|
| 87 | 2022-11-28T20:29:56.485Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 2.469135764s
|
| 88 | 2022-11-28T20:29:56.496Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 89 | 2022-11-28T20:29:56.497Z | WARN | SledAgent: failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 862.109587ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 90 | 2022-11-28T20:29:56.531Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.107745338s
|
| 91 | 2022-11-28T20:29:56.631Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.760520641s
|
| 92 | 2022-11-28T20:29:57.022Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.774480433s
|
| 93 | 2022-11-28T20:29:57.154Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 1.657695886s
|
| 94 | 2022-11-28T20:29:57.241Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 2.09509842s
|
| 95 | 2022-11-28T20:29:57.360Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 96 | 2022-11-28T20:29:57.360Z | WARN | SledAgent: failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 531.367887ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 97 | 2022-11-28T20:29:57.640Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.496795634s
|
| 98 | 2022-11-28T20:29:57.695Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:1::1 exists
|
| 99 | 2022-11-28T20:29:57.695Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:1::1, prefix: 64 }))
zone = oxz_internal_dns
|
| 100 | 2022-11-28T20:29:58.239Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:1::1 exists - OK
|
| 101 | 2022-11-28T20:29:58.239Z | INFO | SledAgent (ServiceManager): GZ addresses: [
fd00:1122:3344:1::2,
]
|
| 102 | 2022-11-28T20:29:58.240Z | INFO | SledAgent (ServiceManager): Ensuring GZ address fd00:1122:3344:1::2 exists
|
| 103 | 2022-11-28T20:29:58.927Z | INFO | SledAgent (ServiceManager): Setting up internal-dns service
|
| 104 | 2022-11-28T20:29:59.033Z | INFO | SledAgent (ServiceManager): Sending prefix to ddmd for advertisement
prefix = Ipv6Prefix { addr: fd00:1122:3344:1::, mask: 64 }
|
| 105 | 2022-11-28T20:29:59.033Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:56173
req_id = d6ed10a0-e8c1-4c89-b311-40d2d4acd4ed
response_code = 204
uri = /services
|
| 106 | 2022-11-28T20:29:59.034Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 107 | 2022-11-28T20:29:59.035Z | INFO | SledAgent (RSS): Adding DNS server: [fd00:1122:3344:1::1]:5353
client = DNS
|
| 108 | 2022-11-28T20:29:59.044Z | INFO | SledAgent (RSS): sending dataset requests...
|
| 109 | 2022-11-28T20:29:59.044Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 09a9a25f-2602-4e2f-9630-31af9c492c3e, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::7]:32345 }
|
| 110 | 2022-11-28T20:29:59.044Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:51513
|
| 111 | 2022-11-28T20:29:59.044Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::7]:32345, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 112 | 2022-11-28T20:29:59.044Z | INFO | SledAgent (StorageManager): Ensuring dataset oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible exists
|
| 113 | 2022-11-28T20:29:59.087Z | INFO | SledAgent (StorageManager): Ensuring zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible is running
|
| 114 | 2022-11-28T20:29:59.095Z | INFO | SledAgent (StorageManager): Zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible was not found
|
| 115 | 2022-11-28T20:29:59.109Z | INFO | SledAgent (StorageManager): Configuring new Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 116 | 2022-11-28T20:29:59.126Z | INFO | SledAgent (StorageManager): Installing Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 117 | 2022-11-28T20:30:00.595Z | INFO | SledAgent (StorageManager): Zone booting
zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 118 | 2022-11-28T20:30:02.144Z | INFO | SledAgent (StorageManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::7, prefix: 64 }))
zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 119 | 2022-11-28T20:30:02.710Z | INFO | SledAgent (StorageManager): Initializing Crucible
|
| 120 | 2022-11-28T20:30:03.179Z | INFO | SledAgent (StorageManager): Zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b with address [fd00:1122:3344:101::7]:32345 is running
|
| 121 | 2022-11-28T20:30:03.179Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51513
req_id = 32b66025-b8a3-448f-9f16-db104dafd8b0
response_code = 204
uri = /filesystem
|
| 122 | 2022-11-28T20:30:03.179Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 2713b37a-3043-4ed5-aaff-f38200e45cfb, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 123 | 2022-11-28T20:30:03.180Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 124 | 2022-11-28T20:30:03.180Z | INFO | SledAgent (StorageManager): Ensuring dataset oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible exists
|
| 125 | 2022-11-28T20:30:03.219Z | INFO | SledAgent (StorageManager): Ensuring zone for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible is running
|
| 126 | 2022-11-28T20:30:03.226Z | INFO | SledAgent (StorageManager): Zone for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible was not found
|
| 127 | 2022-11-28T20:30:03.241Z | INFO | SledAgent (StorageManager): Configuring new Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 128 | 2022-11-28T20:30:03.258Z | INFO | SledAgent (StorageManager): Installing Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 129 | 2022-11-28T20:30:04.036Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 1.414106761s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 130 | 2022-11-28T20:30:04.720Z | INFO | SledAgent (StorageManager): Zone booting
zone = oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 131 | 2022-11-28T20:30:05.451Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 132 | 2022-11-28T20:30:05.452Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 957.314141ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 133 | 2022-11-28T20:30:06.410Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 134 | 2022-11-28T20:30:06.410Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 871.755174ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 135 | 2022-11-28T20:30:06.966Z | INFO | SledAgent (StorageManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::8, prefix: 64 }))
zone = oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 136 | 2022-11-28T20:30:07.539Z | INFO | SledAgent (StorageManager): Initializing Crucible
|
| 137 | 2022-11-28T20:30:08.011Z | INFO | SledAgent (StorageManager): Zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 with address [fd00:1122:3344:101::8]:32345 is running
|
| 138 | 2022-11-28T20:30:08.011Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 139 | 2022-11-28T20:30:08.011Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51513
req_id = 6285051c-2117-4e02-a9b6-68aef7ba72a3
response_code = 204
uri = /filesystem
|
| 140 | 2022-11-28T20:30:08.012Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: ffd16cad-e5d5-495e-9c59-4312a3857d91, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }
|
| 141 | 2022-11-28T20:30:08.012Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 805.145153ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 142 | 2022-11-28T20:30:08.012Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 143 | 2022-11-28T20:30:08.012Z | INFO | SledAgent (StorageManager): Ensuring dataset oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible exists
|
| 144 | 2022-11-28T20:30:08.056Z | INFO | SledAgent (StorageManager): Ensuring zone for oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible is running
|
| 145 | 2022-11-28T20:30:08.064Z | INFO | SledAgent (StorageManager): Zone for oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible was not found
|
| 146 | 2022-11-28T20:30:08.081Z | INFO | SledAgent (StorageManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 147 | 2022-11-28T20:30:08.099Z | INFO | SledAgent (StorageManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 148 | 2022-11-28T20:30:08.818Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 149 | 2022-11-28T20:30:08.818Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 1.0188965s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 150 | 2022-11-28T20:30:09.595Z | INFO | SledAgent (StorageManager): Zone booting
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 151 | 2022-11-28T20:30:09.838Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 152 | 2022-11-28T20:30:09.839Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 924.955814ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 153 | 2022-11-28T20:30:10.765Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 154 | 2022-11-28T20:30:10.766Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 1.478327956s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 155 | 2022-11-28T20:30:11.044Z | INFO | SledAgent (StorageManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::9, prefix: 64 }))
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 156 | 2022-11-28T20:30:11.615Z | INFO | SledAgent (StorageManager): Initializing Crucible
|
| 157 | 2022-11-28T20:30:12.083Z | INFO | SledAgent (StorageManager): Zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 with address [fd00:1122:3344:101::9]:32345 is running
|
| 158 | 2022-11-28T20:30:12.084Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51513
req_id = dd830234-512b-49e5-a711-97245aa181aa
response_code = 204
uri = /filesystem
|
| 159 | 2022-11-28T20:30:12.084Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 8.67405814s
|
| 160 | 2022-11-28T20:30:12.084Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 4d08fc19-3d5f-4f6b-9c48-925f8eac7255, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: CockroachDb { all_addresses: [[fd00:1122:3344:101::3]:32221] }, address: [fd00:1122:3344:101::3]:32221 }
|
| 161 | 2022-11-28T20:30:12.084Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.584197985s
|
| 162 | 2022-11-28T20:30:12.084Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.141054674s
|
| 163 | 2022-11-28T20:30:12.084Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: CockroachDb { all_addresses: [[fd00:1122:3344:101::3]:32221] }, address: [fd00:1122:3344:101::3]:32221, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 164 | 2022-11-28T20:30:12.084Z | INFO | SledAgent (StorageManager): Ensuring dataset oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/cockroachdb exists
|
| 165 | 2022-11-28T20:30:12.125Z | INFO | SledAgent (StorageManager): Ensuring zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/cockroachdb is running
|
| 166 | 2022-11-28T20:30:12.133Z | INFO | SledAgent (StorageManager): Zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/cockroachdb was not found
|
| 167 | 2022-11-28T20:30:12.149Z | INFO | SledAgent (StorageManager): Configuring new Omicron zone: oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 168 | 2022-11-28T20:30:12.166Z | INFO | SledAgent (StorageManager): Installing Omicron zone: oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 169 | 2022-11-28T20:30:12.245Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 170 | 2022-11-28T20:30:12.246Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 732.969969ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 171 | 2022-11-28T20:30:12.980Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 172 | 2022-11-28T20:30:12.981Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 1.210623151s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 173 | 2022-11-28T20:30:14.193Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 174 | 2022-11-28T20:30:14.194Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 1.435534723s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 175 | 2022-11-28T20:30:14.627Z | INFO | SledAgent (StorageManager): Zone booting
zone = oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 176 | 2022-11-28T20:30:15.630Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 177 | 2022-11-28T20:30:15.630Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 1.143684066s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 178 | 2022-11-28T20:30:16.718Z | INFO | SledAgent (StorageManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::3, prefix: 64 }))
zone = oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 179 | 2022-11-28T20:30:17.283Z | INFO | SledAgent (StorageManager): start_zone: Loading CRDB manifest
|
| 180 | 2022-11-28T20:30:17.476Z | INFO | SledAgent (StorageManager): start_zone: setting CRDB's config/listen_addr: [fd00:1122:3344:101::3]:32221
|
| 181 | 2022-11-28T20:30:17.503Z | INFO | SledAgent (StorageManager): start_zone: setting CRDB's config/store
|
| 182 | 2022-11-28T20:30:17.531Z | INFO | SledAgent (StorageManager): start_zone: setting CRDB's config/join_addrs
|
| 183 | 2022-11-28T20:30:17.559Z | INFO | SledAgent (StorageManager): start_zone: refreshing manifest
|
| 184 | 2022-11-28T20:30:17.585Z | INFO | SledAgent (StorageManager): start_zone: enabling CRDB service
|
| 185 | 2022-11-28T20:30:17.610Z | INFO | SledAgent (StorageManager): start_zone: awaiting liveness of CRDB
|
| 186 | 2022-11-28T20:30:17.616Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 187 | 2022-11-28T20:30:17.616Z | WARN | SledAgent (StorageManager): cockroachdb not yet alive
|
| 188 | 2022-11-28T20:30:17.616Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 787.512582ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 189 | 2022-11-28T20:30:17.831Z | WARN | SledAgent (StorageManager): cockroachdb not yet alive
|
| 190 | 2022-11-28T20:30:18.405Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 191 | 2022-11-28T20:30:18.406Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 825.702518ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 192 | 2022-11-28T20:30:18.570Z | WARN | SledAgent (StorageManager): cockroachdb not yet alive
|
| 193 | 2022-11-28T20:30:19.232Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 194 | 2022-11-28T20:30:19.233Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 652.915689ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 195 | 2022-11-28T20:30:19.445Z | WARN | SledAgent (StorageManager): cockroachdb not yet alive
|
| 196 | 2022-11-28T20:30:19.886Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 197 | 2022-11-28T20:30:19.887Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 719.081597ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 198 | 2022-11-28T20:30:20.607Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 199 | 2022-11-28T20:30:20.608Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 1.370361023s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 200 | 2022-11-28T20:30:20.870Z | INFO | SledAgent (StorageManager): CRDB is online
|
| 201 | 2022-11-28T20:30:20.871Z | INFO | SledAgent (StorageManager): Formatting CRDB
|
| 202 | 2022-11-28T20:30:21.980Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 203 | 2022-11-28T20:30:21.981Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 1.452614169s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 204 | 2022-11-28T20:30:23.435Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 205 | 2022-11-28T20:30:23.436Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 1.454391692s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 206 | 2022-11-28T20:30:24.892Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 207 | 2022-11-28T20:30:24.893Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 1.281351337s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 208 | 2022-11-28T20:30:26.175Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 209 | 2022-11-28T20:30:26.175Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 1.432427036s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 210 | 2022-11-28T20:30:27.040Z | INFO | SledAgent (StorageManager): Formatting CRDB - Completed
|
| 211 | 2022-11-28T20:30:27.040Z | INFO | SledAgent (StorageManager): Zone oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b with address [fd00:1122:3344:101::3]:32221 is running
|
| 212 | 2022-11-28T20:30:27.041Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51513
req_id = e076a42f-76e2-4e2c-bac8-da2f48f7efa3
response_code = 204
uri = /filesystem
|
| 213 | 2022-11-28T20:30:27.041Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 11.601582476s
|
| 214 | 2022-11-28T20:30:27.041Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 194.513245ms
|
| 215 | 2022-11-28T20:30:27.041Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: a3505b41-a592-420b-84f2-3d76bf0e0a81, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Clickhouse, address: [fd00:1122:3344:101::6]:8123 }
|
| 216 | 2022-11-28T20:30:27.041Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Clickhouse, address: [fd00:1122:3344:101::6]:8123, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 217 | 2022-11-28T20:30:27.041Z | INFO | SledAgent (StorageManager): Ensuring dataset oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/clickhouse exists
|
| 218 | 2022-11-28T20:30:27.095Z | INFO | SledAgent (StorageManager): Ensuring zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/clickhouse is running
|
| 219 | 2022-11-28T20:30:27.104Z | INFO | SledAgent (StorageManager): Zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/clickhouse was not found
|
| 220 | 2022-11-28T20:30:27.120Z | INFO | SledAgent (StorageManager): Configuring new Omicron zone: oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 221 | 2022-11-28T20:30:27.138Z | INFO | SledAgent (StorageManager): Installing Omicron zone: oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 222 | 2022-11-28T20:30:27.609Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 223 | 2022-11-28T20:30:27.610Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 833.175091ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 224 | 2022-11-28T20:30:28.443Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 225 | 2022-11-28T20:30:28.476Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 1.225180062s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 226 | 2022-11-28T20:30:29.554Z | INFO | SledAgent (StorageManager): Zone booting
zone = oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 227 | 2022-11-28T20:30:29.702Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 228 | 2022-11-28T20:30:29.703Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 884.113101ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 229 | 2022-11-28T20:30:30.587Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 230 | 2022-11-28T20:30:30.588Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 1.437980741s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 231 | 2022-11-28T20:30:31.692Z | INFO | SledAgent (StorageManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::6, prefix: 64 }))
zone = oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 232 | 2022-11-28T20:30:32.259Z | INFO | SledAgent (StorageManager): Initialiting Clickhouse
|
| 233 | 2022-11-28T20:30:32.563Z | INFO | SledAgent (StorageManager): Zone oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b with address [fd00:1122:3344:101::6]:8123 is running
|
| 234 | 2022-11-28T20:30:32.564Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 235 | 2022-11-28T20:30:32.564Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51513
req_id = 4731d7ca-cd0f-476e-a9d7-3f1a8f014028
response_code = 204
uri = /filesystem
|
| 236 | 2022-11-28T20:30:32.564Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 1.426834965s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 237 | 2022-11-28T20:30:32.565Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 343.703729ms
|
| 238 | 2022-11-28T20:30:32.566Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 5.164469926s
|
| 239 | 2022-11-28T20:30:32.566Z | INFO | SledAgent (RSS): Inserting DNS record: Backend(Crucible, 2713b37a-3043-4ed5-aaff-f38200e45cfb)
client = DNS
|
| 240 | 2022-11-28T20:30:32.566Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 200.629695ms
|
| 241 | 2022-11-28T20:30:32.566Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 428.565059ms
|
| 242 | 2022-11-28T20:30:32.567Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 168.085455ms
|
| 243 | 2022-11-28T20:30:32.567Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 270.975707ms
|
| 244 | 2022-11-28T20:30:32.568Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 9.66119395s
|
| 245 | 2022-11-28T20:30:32.568Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 6.632058981s
|
| 246 | 2022-11-28T20:30:32.568Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 10.335304416s
|
| 247 | 2022-11-28T20:30:32.569Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 3.797246005s
|
| 248 | 2022-11-28T20:30:32.569Z | INFO | SledAgent (RSS): Inserting DNS record: Backend(Crucible, 09a9a25f-2602-4e2f-9630-31af9c492c3e)
client = DNS
|
| 249 | 2022-11-28T20:30:32.569Z | INFO | SledAgent (RSS): Inserting DNS record: Service(Cockroach)
client = DNS
|
| 250 | 2022-11-28T20:30:32.570Z | INFO | SledAgent (RSS): Inserting DNS record: Service(Clickhouse)
client = DNS
|
| 251 | 2022-11-28T20:30:32.570Z | INFO | SledAgent (RSS): Inserting DNS record: Backend(Crucible, ffd16cad-e5d5-495e-9c59-4312a3857d91)
client = DNS
|
| 252 | 2022-11-28T20:30:32.570Z | INFO | SledAgent (RSS): Finished setting up agents and datasets
|
| 253 | 2022-11-28T20:30:32.575Z | INFO | SledAgent (RSS): sending service requests...
|
| 254 | 2022-11-28T20:30:32.575Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { id: e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c, zone_type: Nexus, addresses: [fd00:1122:3344:101::4], gz_addresses: [], services: [Nexus { internal_ip: fd00:1122:3344:101::4, external_ip: 192.168.1.20 }] }, ServiceZoneRequest { id: 1da65e5b-210c-4859-a7d7-200c1e659972, zone_type: Oximeter, addresses: [fd00:1122:3344:101::5], gz_addresses: [], services: [Oximeter] }, ServiceZoneRequest { id: b8568fdf-8b3e-40d6-ae1a-8737e10bc1dc, zone_type: InternalDNS, addresses: [fd00:1122:3344:1::1], gz_addresses: [fd00:1122:3344:1::2], services: [InternalDns { server_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53 }] }]
|
| 255 | 2022-11-28T20:30:32.575Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:33868
|
| 256 | 2022-11-28T20:30:32.575Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Oximeter
|
| 257 | 2022-11-28T20:30:32.575Z | INFO | SledAgent (ServiceManager): Service zone oximeter does not yet exist
|
| 258 | 2022-11-28T20:30:32.591Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_oximeter
|
| 259 | 2022-11-28T20:30:32.608Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_oximeter
|
| 260 | 2022-11-28T20:30:32.736Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 428.943346ms
|
| 261 | 2022-11-28T20:30:32.768Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 352.777501ms
|
| 262 | 2022-11-28T20:30:32.840Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 747.226442ms
|
| 263 | 2022-11-28T20:30:32.910Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 291.845067ms
|
| 264 | 2022-11-28T20:30:32.997Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.249681198s
|
| 265 | 2022-11-28T20:30:33.122Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 543.583011ms
|
| 266 | 2022-11-28T20:30:33.167Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 720.78835ms
|
| 267 | 2022-11-28T20:30:33.203Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 579.550139ms
|
| 268 | 2022-11-28T20:30:33.540Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_oximeter
|
| 269 | 2022-11-28T20:30:33.589Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.250689749s
|
| 270 | 2022-11-28T20:30:33.667Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.415950978s
|
| 271 | 2022-11-28T20:30:33.785Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.213365708s
|
| 272 | 2022-11-28T20:30:33.889Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.377072895s
|
| 273 | 2022-11-28T20:30:33.992Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 274 | 2022-11-28T20:30:33.992Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 798.801734ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 275 | 2022-11-28T20:30:34.248Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.416923378s
|
| 276 | 2022-11-28T20:30:34.791Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 277 | 2022-11-28T20:30:34.792Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 574.114979ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 278 | 2022-11-28T20:30:34.842Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.081993668s
|
| 279 | 2022-11-28T20:30:34.999Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 3.752350669s
|
| 280 | 2022-11-28T20:30:35.268Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 4.782835542s
|
| 281 | 2022-11-28T20:30:35.367Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 282 | 2022-11-28T20:30:35.368Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 1.227056464s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 283 | 2022-11-28T20:30:35.796Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::5 exists
|
| 284 | 2022-11-28T20:30:35.796Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::5, prefix: 64 }))
zone = oxz_oximeter
|
| 285 | 2022-11-28T20:30:36.340Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::5 exists - OK
|
| 286 | 2022-11-28T20:30:36.342Z | INFO | SledAgent (ServiceManager): GZ addresses: []
|
| 287 | 2022-11-28T20:30:36.555Z | INFO | SledAgent (ServiceManager): Setting up oximeter service
|
| 288 | 2022-11-28T20:30:36.660Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Nexus
|
| 289 | 2022-11-28T20:30:36.662Z | INFO | SledAgent (ServiceManager): Service zone nexus does not yet exist
|
| 290 | 2022-11-28T20:30:36.677Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 291 | 2022-11-28T20:30:36.677Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 1.482127488s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 292 | 2022-11-28T20:30:36.678Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 3.207316955s
|
| 293 | 2022-11-28T20:30:36.678Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 9.654771396s
|
| 294 | 2022-11-28T20:30:36.678Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 4.255303022s
|
| 295 | 2022-11-28T20:30:36.679Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 4.399855223s
|
| 296 | 2022-11-28T20:30:36.682Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus
|
| 297 | 2022-11-28T20:30:36.698Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus
|
| 298 | 2022-11-28T20:30:37.944Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 7.936307666s
|
| 299 | 2022-11-28T20:30:38.128Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_nexus
|
| 300 | 2022-11-28T20:30:38.161Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 301 | 2022-11-28T20:30:38.162Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 1.451379296s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 302 | 2022-11-28T20:30:38.645Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 14.667099328s
|
| 303 | 2022-11-28T20:30:38.754Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 11.506167432s
|
| 304 | 2022-11-28T20:30:39.201Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 10.422577596s
|
| 305 | 2022-11-28T20:30:39.613Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 306 | 2022-11-28T20:30:39.614Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 1.265659035s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 307 | 2022-11-28T20:30:39.887Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 6.871782726s
|
| 308 | 2022-11-28T20:30:40.053Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 8.239073891s
|
| 309 | 2022-11-28T20:30:40.881Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 310 | 2022-11-28T20:30:40.882Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 520.8684ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 311 | 2022-11-28T20:30:40.936Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 10.734021211s
|
| 312 | 2022-11-28T20:30:41.080Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 5.518743715s
|
| 313 | 2022-11-28T20:30:41.404Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 314 | 2022-11-28T20:30:41.405Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 757.223139ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 315 | 2022-11-28T20:30:42.164Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 316 | 2022-11-28T20:30:42.165Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 527.338373ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 317 | 2022-11-28T20:30:42.201Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::4 exists
|
| 318 | 2022-11-28T20:30:42.201Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::4, prefix: 64 }))
zone = oxz_nexus
|
| 319 | 2022-11-28T20:30:42.754Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::4 exists - OK
|
| 320 | 2022-11-28T20:30:42.754Z | INFO | SledAgent (ServiceManager): GZ addresses: []
|
| 321 | 2022-11-28T20:30:42.915Z | INFO | SledAgent (ServiceManager): Setting up Nexus service
|
| 322 | 2022-11-28T20:30:42.915Z | INFO | SledAgent (ServiceManager): Adding address: Static(V4(Ipv4Network { addr: 192.168.1.20, prefix: 24 }))
zone = oxz_nexus
|
| 323 | 2022-11-28T20:30:43.503Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 324 | 2022-11-28T20:30:43.507Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 43.376161058s
|
| 325 | 2022-11-28T20:30:43.507Z | WARN | SledAgent: failed to notify nexus about sled agent: no record found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: AAAA, query_class: IN }, will retry in 972.774054ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 326 | 2022-11-28T20:30:43.507Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 9.684464226s
|
| 327 | 2022-11-28T20:30:43.531Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:33868
req_id = 8fa680a9-e070-459e-b6fe-e057e185aaae
response_code = 204
uri = /services
|
| 328 | 2022-11-28T20:30:43.533Z | INFO | SledAgent (RSS): Inserting DNS record: Service(Oximeter)
client = DNS
|
| 329 | 2022-11-28T20:30:43.533Z | INFO | SledAgent (RSS): Inserting DNS record: Service(InternalDNS)
client = DNS
|
| 330 | 2022-11-28T20:30:43.533Z | INFO | SledAgent (RSS): Inserting DNS record: Service(Nexus)
client = DNS
|
| 331 | 2022-11-28T20:30:43.534Z | INFO | SledAgent (RSS): Finished setting up services
|
| 332 | 2022-11-28T20:30:44.481Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 333 | 2022-11-28T20:30:44.487Z | WARN | SledAgent: failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 1.339707251s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 334 | 2022-11-28T20:30:45.828Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 335 | 2022-11-28T20:30:45.834Z | WARN | SledAgent: failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 1.285005236s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 336 | 2022-11-28T20:30:45.886Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 17.12628522s
|
| 337 | 2022-11-28T20:30:46.340Z | WARN | SledAgent (StorageManager): failed to notify nexus, will retry in 22.87957163s
|
| 338 | 2022-11-28T20:30:46.605Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 17.757926365s
|
| 339 | 2022-11-28T20:30:46.765Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 21.882761489s
|
| 340 | 2022-11-28T20:30:47.121Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 341 | 2022-11-28T20:30:47.127Z | WARN | SledAgent: failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 608.085686ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 342 | 2022-11-28T20:30:47.736Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 343 | 2022-11-28T20:30:47.741Z | WARN | SledAgent: failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 575.182059ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 344 | 2022-11-28T20:30:48.298Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 21.877519545s
|
| 345 | 2022-11-28T20:30:48.317Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 346 | 2022-11-28T20:30:48.322Z | WARN | SledAgent: failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 1.056417525s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 347 | 2022-11-28T20:30:49.380Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 348 | 2022-11-28T20:30:49.385Z | WARN | SledAgent: failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 949.322082ms
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 349 | 2022-11-28T20:30:50.266Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 10.707599704s
|
| 350 | 2022-11-28T20:30:50.336Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 351 | 2022-11-28T20:30:50.341Z | WARN | SledAgent: failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 1.090547171s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 352 | 2022-11-28T20:30:51.433Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 353 | 2022-11-28T20:30:51.438Z | WARN | SledAgent: failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 1.060754483s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 354 | 2022-11-28T20:30:51.676Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 10.733869154s
|
| 355 | 2022-11-28T20:30:52.500Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 356 | 2022-11-28T20:30:52.505Z | WARN | SledAgent: failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 1.166583447s
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 357 | 2022-11-28T20:30:53.672Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 358 | 2022-11-28T20:30:53.681Z | INFO | SledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
|
| 359 | 2022-11-28T20:31:00.985Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 18.34026616s
|
| 360 | 2022-11-28T20:31:02.420Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 36.643679871s
|
| 361 | 2022-11-28T20:31:04.373Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 17.438215291s
|
| 362 | 2022-11-28T20:31:19.334Z | WARN | SledAgent (StorageManager): failed to notify nexus about datasets, will retry in 95.653425626s
|