Max level shown:
1 [ Aug 17 20:58:07 Enabled. ]
2 [ Aug 17 20:58:07 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ]
3 [ Aug 17 20:58:07 Method "start" exited with status 0. ]
4 [ Aug 17 20:58:07 Rereading configuration. ]
5 [ Aug 17 20:58:07 No 'refresh' method defined. Treating as :true. ]
6 note: configured to log to "/dev/stdout"
72022-08-17T20:58:07.698ZINFOSledAgent: Starting mg-ddm service
82022-08-17T20:58:07.698ZINFOSledAgent: Importing mg-ddm service path = /opt/oxide/mg-ddm/pkg/ddm/manifest.xml
92022-08-17T20:58:07.804ZINFOSledAgent: Setting mg-ddm interface interface = net0/linklocal
102022-08-17T20:58:07.812ZINFOSledAgent: Enabling mg-ddm service
112022-08-17T20:58:07.819ZINFOSledAgent: detecting (real or simulated) SP
122022-08-17T20:58:07.819ZINFOSledAgent: setting up bootstrap agent server
132022-08-17T20:58:07.819ZINFOSledAgent: Ensuring config directory exists path = /var/oxide
142022-08-17T20:58:08.322ZINFOSledAgent (BootstrapAgentServer): Started listening local_addr = [fdb0:18c0:4d0d:9fb2::1]:12346
152022-08-17T20:58:08.322ZINFOSledAgent (BootstrapAgent): bootstrap service initializing server = fb0f7546-4d46-40ca-9d56-cbb810684ca7
162022-08-17T20:58:08.322ZINFOSledAgent: Sending prefix to ddmd for advertisement prefix = Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, mask: 64 }
172022-08-17T20:58:08.323ZINFOSledAgent (RSS): Injecting RSS configuration: SetupServiceConfig { rack_subnet: fd00:1122:3344:100::, requests: [ HardcodedSledRequest { datasets: [ DatasetEnsureBody { id: 09a9a25f26024e2f963031af9c492c3e, zpool_id: d462a7f7b62840fe80ff4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::6]:32345, }, DatasetEnsureBody { id: 2713b37a30434ed5aafff38200e45cfb, zpool_id: e4b4dc87ab4649fba4b4d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::7]:32345, }, DatasetEnsureBody { id: ffd16cade5d5495e9c594312a3857d91, zpool_id: f4b4dc87ab4649fba4b4d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345, }, DatasetEnsureBody { id: 4d08fc193d5f4f6b9c48925f8eac7255, zpool_id: d462a7f7b62840fe80ff4e4189e2d62b, dataset_kind: CockroachDb { all_addresses: [ [fd00:1122:3344:101::2]:32221, ], }, address: [fd00:1122:3344:101::2]:32221, }, DatasetEnsureBody { id: a3505b41a592420b84f23d76bf0e0a81, zpool_id: d462a7f7b62840fe80ff4e4189e2d62b, dataset_kind: Clickhouse, address: [fd00:1122:3344:101::5]:8123, }, ], services: [ ServiceRequest { id: e6bff1ff24fb49dca54ec6a350cd4d6c, name: "nexus", addresses: [ fd00:1122:3344:101::3, ], gz_addresses: [], service_type: Nexus { internal_ip: fd00:1122:3344:101::3, external_ip: 192.168.1.20, }, }, ServiceRequest { id: 1da65e5b210c4859a7d7200c1e659972, name: "oximeter", addresses: [ fd00:1122:3344:101::4, ], gz_addresses: [], service_type: Oximeter, }, ServiceRequest { id: a0fe5ebc92616f77acc1972481755789, name: "dendrite", addresses: [ fd00:1122:3344:101::9, ], gz_addresses: [], service_type: Dendrite { asic: TofinoStub, }, }, ], dns_services: [], }, ], rack_secret_threshold: 1, gateway: Gateway { address: None, mac: MacAddr6( [ 0, 13, 185, 84, 254, 228, ], ), }, }
182022-08-17T20:58:08.323ZINFOSledAgent (RSS): RSS configuration has not been fully applied yet
192022-08-17T20:58:08.329ZINFOSledAgent (RSS): Received prefixes from ddmd prefixes = {}
202022-08-17T20:58:08.329ZINFOSledAgent (RSS): Enough peers exist to enact RSS plan
212022-08-17T20:58:08.330ZINFOSledAgent (RSS): Creating new allocation plan
222022-08-17T20:58:08.330ZINFOSledAgent (RSS): dns_subnets: [ DnsSubnet { subnet: Ipv6Subnet { net: Ipv6Net( Ipv6Network { addr: fd00:1122:3344:1::, prefix: 64, }, ), }, }, ]
232022-08-17T20:58:08.330ZINFOSledAgent (RSS): Serializing plan
242022-08-17T20:58:08.330ZINFOSledAgent (RSS): Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1
252022-08-17T20:58:08.330ZINFOSledAgent (RSS): Plan serialized as: ["[fdb0:18c0:4d0d:9fb2::1]:12346".initialization_request] id = "5e049c3e-9621-48f2-a10c-7c4c53ae41a3" rack_id = "7de63f8e-63a6-403e-ae13-3cdc2f407ab8" ["[fdb0:18c0:4d0d:9fb2::1]:12346".initialization_request.gateway] mac = [0, 13, 185, 84, 254, 228] ["[fdb0:18c0:4d0d:9fb2::1]:12346".initialization_request.subnet] net = "fd00:1122:3344:101::/64" [["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset]] address = "[fd00:1122:3344:101::6]:32345" id = "09a9a25f-2602-4e2f-9630-31af9c492c3e" zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b" ["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset.dataset_kind] type = "crucible" [["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset]] address = "[fd00:1122:3344:101::7]:32345" id = "2713b37a-3043-4ed5-aaff-f38200e45cfb" zpool_id = "e4b4dc87-ab46-49fb-a4b4-d361ae214c03" ["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset.dataset_kind] type = "crucible" [["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset]] address = "[fd00:1122:3344:101::8]:32345" id = "ffd16cad-e5d5-495e-9c59-4312a3857d91" zpool_id = "f4b4dc87-ab46-49fb-a4b4-d361ae214c03" ["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset.dataset_kind] type = "crucible" [["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset]] address = "[fd00:1122:3344:101::2]:32221" id = "4d08fc19-3d5f-4f6b-9c48-925f8eac7255" zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b" ["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset.dataset_kind] all_addresses = ["[fd00:1122:3344:101::2]:32221"] type = "cockroach_db" [["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset]] address = "[fd00:1122:3344:101::5]:8123" id = "a3505b41-a592-420b-84f2-3d76bf0e0a81" zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b" ["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dataset.dataset_kind] type = "clickhouse" [["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dns_service]] addresses = ["fd00:1122:3344:1::1"] gz_addresses = ["fd00:1122:3344:1::2"] id = "4b51d2cc-ccf1-49ea-b3bb-0463bb1cff8b" name = "internal-dns" ["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.dns_service.service_type] dns_address = "[fd00:1122:3344:1::1]:53" server_address = "[fd00:1122:3344:1::1]:5353" type = "internal_dns" [["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.service]] addresses = ["fd00:1122:3344:101::3"] gz_addresses = [] id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c" name = "nexus" ["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.service.service_type] external_ip = "192.168.1.20" internal_ip = "fd00:1122:3344:101::3" type = "nexus" [["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.service]] addresses = ["fd00:1122:3344:101::4"] gz_addresses = [] id = "1da65e5b-210c-4859-a7d7-200c1e659972" name = "oximeter" ["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.service.service_type] type = "oximeter" [["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.service]] addresses = ["fd00:1122:3344:101::9"] gz_addresses = [] id = "a0fe5ebc-9261-6f77-acc1-972481755789" name = "dendrite" ["[fdb0:18c0:4d0d:9fb2::1]:12346".services_request.service.service_type] asic = "tofino_stub" type = "dendrite"
262022-08-17T20:58:08.330ZINFOSledAgent (RSS): Plan written to storage
272022-08-17T20:58:08.330ZINFOSledAgent (RSS): Skipping rack secret creation (only one sled present)
282022-08-17T20:58:08.330ZINFOSledAgent (BootstrapAgentRssHandler): Received initialization request from RSS request = SledAgentRequest { id: 5e049c3e-9621-48f2-a10c-7c4c53ae41a3, rack_id: 7de63f8e-63a6-403e-ae13-3cdc2f407ab8, gateway: Gateway { address: None, mac: MacAddr6([0, 13, 185, 84, 254, 228]) }, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } target_sled = [fdb0:18c0:4d0d:9fb2::1]:12346
292022-08-17T20:58:08.330ZINFOSledAgent (BootstrapAgentServer): Accepted connection remote_addr = [fdb0:18c0:4d0d:9fb2::1]:45495
302022-08-17T20:58:08.330ZINFOSledAgent (BootstrapAgentRssHandler): No SP available; proceeding without sprockets auth BootstrapAgentClient = [fdb0:18c0:4d0d:9fb2::1]:12346
312022-08-17T20:58:08.330ZINFOSledAgent (BootstrapAgentServer): No SP available; proceeding without sprockets auth remote_addr = [fdb0:18c0:4d0d:9fb2::1]:45495
322022-08-17T20:58:08.330ZINFOSledAgent (BootstrapAgent): Loading Sled Agent: SledAgentRequest { id: 5e049c3e-9621-48f2-a10c-7c4c53ae41a3, rack_id: 7de63f8e-63a6-403e-ae13-3cdc2f407ab8, gateway: Gateway { address: None, mac: MacAddr6([0, 13, 185, 84, 254, 228]) }, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } server = fb0f7546-4d46-40ca-9d56-cbb810684ca7
332022-08-17T20:58:08.330ZINFOSledAgent: setting up sled agent server
342022-08-17T20:58:08.330ZINFOSledAgent: created sled agent sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
352022-08-17T20:58:08.822ZINFOSledAgent: updating xde driver configuration file for external IP workaround conf_file = "/kernel/drv/xde.conf" sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
362022-08-17T20:58:08.865ZINFOSledAgent: using '[AddrObject { interface: "net0", name: "linklocal" }, AddrObject { interface: "net1", name: "linklocal" }]' as data links for xde driver sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
372022-08-17T20:58:08.992ZINFOSledAgent: Sled Agent upserting zpool to Storage Manager: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
382022-08-17T20:58:08.999ZINFOSledAgent: Sled Agent upserting zpool to Storage Manager: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
392022-08-17T20:58:09.007ZINFOSledAgent: Sled Agent upserting zpool to Storage Manager: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 sled_id = fb0f7546-4d46-40ca-9d56-cbb810684ca7
402022-08-17T20:58:09.031ZINFOSledAgent (ServiceManager): No sled services found at /var/oxide/service.toml
412022-08-17T20:58:09.031ZINFOSledAgent (dropshot (SledAgent)): listening local_addr = [fd00:1122:3344:101::1]:12345
422022-08-17T20:58:09.031ZINFOSledAgent (BootstrapAgent): Sled Agent loaded; recording configuration server = fb0f7546-4d46-40ca-9d56-cbb810684ca7
432022-08-17T20:58:09.031ZINFOSledAgent (StorageManager): Storage manager processing zpool: ZpoolInfo { name: "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b", size: 10200547328, allocated: 127488, free: 10200419840, health: Online, }
442022-08-17T20:58:09.031ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
452022-08-17T20:58:09.034ZINFOSledAgent: Sending prefix to ddmd for advertisement prefix = Ipv6Prefix { addr: fd00:1122:3344:101::, mask: 64 }
462022-08-17T20:58:09.034ZINFOSledAgent (BootstrapAgentServer): Connection closed remote_addr = [fdb0:18c0:4d0d:9fb2::1]:45495
472022-08-17T20:58:09.034ZINFOSledAgent (BootstrapAgentRssHandler): Peer agent initialized peer = [fdb0:18c0:4d0d:9fb2::1]:12346
482022-08-17T20:58:09.034ZINFOSledAgent (BootstrapAgentRssHandler): Initialized sled agent target_sled = [fdb0:18c0:4d0d:9fb2::1]:12346
492022-08-17T20:58:09.035ZWARNSledAgent: failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 190.483085ms
502022-08-17T20:58:09.039ZINFOSledAgent (StorageManager): Storage manager processing zpool: ZpoolInfo { name: "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03", size: 10200547328, allocated: 127488, free: 10200419840, health: Online, }
512022-08-17T20:58:09.043ZINFOSledAgent (RSS): sending service requests...
522022-08-17T20:58:09.043ZINFOSledAgent (RSS): initializing sled services: [ServiceRequest { id: 4b51d2cc-ccf1-49ea-b3bb-0463bb1cff8b, name: "internal-dns", addresses: [fd00:1122:3344:1::1], gz_addresses: [fd00:1122:3344:1::2], service_type: InternalDns { server_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53 } }]
532022-08-17T20:58:09.043ZINFOSledAgent (dropshot (SledAgent)): accepted connection local_addr = [fd00:1122:3344:101::1]:12345 remote_addr = [fd00:1122:3344:101::1]:45829
542022-08-17T20:58:09.043ZINFOSledAgent (ServiceManager): Ensuring service is initialized: ServiceRequest { id: 4b51d2cc-ccf1-49ea-b3bb-0463bb1cff8b, name: "internal-dns", addresses: [fd00:1122:3344:1::1], gz_addresses: [fd00:1122:3344:1::2], service_type: InternalDns { server_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53 } }
552022-08-17T20:58:09.043ZINFOSledAgent (ServiceManager): Service internal-dns does not yet exist
562022-08-17T20:58:09.048ZINFOSledAgent (StorageManager): Storage manager processing zpool: ZpoolInfo { name: "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03", size: 10200547328, allocated: 563712, free: 10199983616, health: Online, }
572022-08-17T20:58:09.057ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 160.848712ms
582022-08-17T20:58:09.057ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 260.118052ms
592022-08-17T20:58:09.057ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 271.957022ms
602022-08-17T20:58:09.057ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 181.815006ms
612022-08-17T20:58:09.057ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 288.702761ms
622022-08-17T20:58:09.057ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 303.389995ms
632022-08-17T20:58:09.059ZINFOSledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal-dns
642022-08-17T20:58:09.075ZINFOSledAgent (ServiceManager): Installing Omicron zone: oxz_internal-dns
652022-08-17T20:58:09.220ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 373.53334ms
662022-08-17T20:58:09.226ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
672022-08-17T20:58:09.226ZWARNSledAgent: failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 356.734703ms
682022-08-17T20:58:09.241ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 430.482854ms
692022-08-17T20:58:09.319ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 608.014163ms
702022-08-17T20:58:09.330ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 569.162412ms
712022-08-17T20:58:09.348ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 710.463436ms
722022-08-17T20:58:09.362ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 701.899657ms
732022-08-17T20:58:09.584ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
742022-08-17T20:58:09.584ZWARNSledAgent: failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 582.824299ms
752022-08-17T20:58:09.595ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 555.565087ms
762022-08-17T20:58:09.673ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 1.056384132s
772022-08-17T20:58:09.900ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.079211077s
782022-08-17T20:58:09.929ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 1.448080707s
792022-08-17T20:58:10.060ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 1.252593132s
802022-08-17T20:58:10.065ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 870.264992ms
812022-08-17T20:58:10.152ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.046919268s
822022-08-17T20:58:10.155ZINFOSledAgent (ServiceManager): Zone booting zone = oxz_internal-dns
832022-08-17T20:58:10.169ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
842022-08-17T20:58:10.169ZWARNSledAgent: failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 773.978468ms
852022-08-17T20:58:10.731ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 1.167801443s
862022-08-17T20:58:10.936ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.869552801s
872022-08-17T20:58:10.944ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
882022-08-17T20:58:10.945ZWARNSledAgent: failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 913.216395ms
892022-08-17T20:58:10.981ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.747880958s
902022-08-17T20:58:11.314ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 2.158796293s
912022-08-17T20:58:11.378ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 2.628166166s
922022-08-17T20:58:11.859ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
932022-08-17T20:58:11.859ZWARNSledAgent: failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 1.427510254s
942022-08-17T20:58:11.900ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 2.336974445s
952022-08-17T20:58:11.923ZINFOSledAgent (ServiceManager): Ensuring address fd00:1122:3344:1::1 exists
962022-08-17T20:58:11.923ZINFOSledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:1::1, prefix: 64 })) zone = oxz_internal-dns
972022-08-17T20:58:12.467ZINFOSledAgent (ServiceManager): Ensuring address fd00:1122:3344:1::1 exists - OK
982022-08-17T20:58:12.467ZINFOSledAgent (ServiceManager): GZ addresses: [ fd00:1122:3344:1::2, ]
992022-08-17T20:58:12.467ZINFOSledAgent (ServiceManager): Ensuring GZ address fd00:1122:3344:1::2 exists
1002022-08-17T20:58:13.152ZINFOSledAgent (ServiceManager): Setting up internal-dns service
1012022-08-17T20:58:13.255ZINFOSledAgent (ServiceManager): Sending prefix to ddmd for advertisement prefix = Ipv6Prefix { addr: fd00:1122:3344:1::, mask: 64 }
1022022-08-17T20:58:13.255ZINFOSledAgent (dropshot (SledAgent)): request completed local_addr = [fd00:1122:3344:101::1]:12345 method = PUT remote_addr = [fd00:1122:3344:101::1]:45829 req_id = 6e2af4e9-1b46-47ce-8090-df3c43c87b64 response_code = 204 uri = /services
1032022-08-17T20:58:13.257ZINFOSledAgent (RSS): Adding DNS server: [fd00:1122:3344:1::1]:5353 client = DNS
1042022-08-17T20:58:13.266ZINFOSledAgent (RSS): sending dataset requests...
1052022-08-17T20:58:13.266ZINFOSledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 09a9a25f-2602-4e2f-9630-31af9c492c3e, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::6]:32345 }
1062022-08-17T20:58:13.266ZINFOSledAgent (dropshot (SledAgent)): accepted connection local_addr = [fd00:1122:3344:101::1]:12345 remote_addr = [fd00:1122:3344:101::1]:35538
1072022-08-17T20:58:13.266ZINFOSledAgent (StorageManager): add_dataset: NewFilesystemRequest { zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::6]:32345, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
1082022-08-17T20:58:13.266ZINFOSledAgent (StorageManager): Ensuring dataset oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible exists
1092022-08-17T20:58:13.307ZINFOSledAgent (StorageManager): Ensuring zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible is running
1102022-08-17T20:58:13.314ZINFOSledAgent (StorageManager): Zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible was not found
1112022-08-17T20:58:13.328ZINFOSledAgent (StorageManager): Configuring new Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
1122022-08-17T20:58:13.345ZINFOSledAgent (StorageManager): Installing Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
1132022-08-17T20:58:14.452ZINFOSledAgent (StorageManager): Zone booting zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
1142022-08-17T20:58:14.568ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
1152022-08-17T20:58:14.570ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.234413574s
1162022-08-17T20:58:15.806ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
1172022-08-17T20:58:15.806ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.088187175s
1182022-08-17T20:58:16.585ZINFOSledAgent (StorageManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::6, prefix: 64 })) zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
1192022-08-17T20:58:17.153ZINFOSledAgent (StorageManager): Initializing Crucible
1202022-08-17T20:58:17.610ZINFOSledAgent (StorageManager): Zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b with address [fd00:1122:3344:101::6]:32345 is running
1212022-08-17T20:58:17.610ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
1222022-08-17T20:58:17.611ZINFOSledAgent (dropshot (SledAgent)): request completed local_addr = [fd00:1122:3344:101::1]:12345 method = PUT remote_addr = [fd00:1122:3344:101::1]:35538 req_id = ef1cd930-fbd9-4b2b-83ef-c0830878dbf4 response_code = 204 uri = /filesystem
1232022-08-17T20:58:17.611ZINFOSledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 2713b37a-3043-4ed5-aaff-f38200e45cfb, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::7]:32345 }
1242022-08-17T20:58:17.611ZINFOSledAgent (StorageManager): add_dataset: NewFilesystemRequest { zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::7]:32345, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
1252022-08-17T20:58:17.611ZINFOSledAgent (StorageManager): Ensuring dataset oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible exists
1262022-08-17T20:58:17.613ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 530.822844ms
1272022-08-17T20:58:17.652ZINFOSledAgent (StorageManager): Ensuring zone for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible is running
1282022-08-17T20:58:17.660ZINFOSledAgent (StorageManager): Zone for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible was not found
1292022-08-17T20:58:17.673ZINFOSledAgent (StorageManager): Configuring new Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
1302022-08-17T20:58:17.690ZINFOSledAgent (StorageManager): Installing Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
1312022-08-17T20:58:18.146ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
1322022-08-17T20:58:18.147ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.151460123s
1332022-08-17T20:58:18.796ZINFOSledAgent (StorageManager): Zone booting zone = oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
1342022-08-17T20:58:19.300ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
1352022-08-17T20:58:19.300ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 779.036193ms
1362022-08-17T20:58:20.080ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
1372022-08-17T20:58:20.081ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 826.247023ms
1382022-08-17T20:58:20.908ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
1392022-08-17T20:58:20.909ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 574.788598ms
1402022-08-17T20:58:21.033ZINFOSledAgent (StorageManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::7, prefix: 64 })) zone = oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
1412022-08-17T20:58:21.600ZINFOSledAgent (StorageManager): Initializing Crucible
1422022-08-17T20:58:22.064ZINFOSledAgent (StorageManager): Zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 with address [fd00:1122:3344:101::7]:32345 is running
1432022-08-17T20:58:22.064ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
1442022-08-17T20:58:22.065ZINFOSledAgent (dropshot (SledAgent)): request completed local_addr = [fd00:1122:3344:101::1]:12345 method = PUT remote_addr = [fd00:1122:3344:101::1]:35538 req_id = e071a26d-2de1-4373-8109-f484049fe739 response_code = 204 uri = /filesystem
1452022-08-17T20:58:22.065ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 4.314968052s
1462022-08-17T20:58:22.065ZINFOSledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: ffd16cad-e5d5-495e-9c59-4312a3857d91, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
1472022-08-17T20:58:22.065ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 3.317223484s
1482022-08-17T20:58:22.065ZINFOSledAgent (StorageManager): add_dataset: NewFilesystemRequest { zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
1492022-08-17T20:58:22.065ZINFOSledAgent (StorageManager): Ensuring dataset oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible exists
1502022-08-17T20:58:22.068ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 873.171137ms
1512022-08-17T20:58:22.106ZINFOSledAgent (StorageManager): Ensuring zone for oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible is running
1522022-08-17T20:58:22.113ZINFOSledAgent (StorageManager): Zone for oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible was not found
1532022-08-17T20:58:22.125ZINFOSledAgent (StorageManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
1542022-08-17T20:58:22.141ZINFOSledAgent (StorageManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
1552022-08-17T20:58:22.942ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
1562022-08-17T20:58:22.943ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.176810436s
1572022-08-17T20:58:23.263ZINFOSledAgent (StorageManager): Zone booting zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
1582022-08-17T20:58:24.122ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
1592022-08-17T20:58:24.123ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 546.071049ms
1602022-08-17T20:58:24.669ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
1612022-08-17T20:58:24.670ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 944.545224ms
1622022-08-17T20:58:25.055ZINFOSledAgent (StorageManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::8, prefix: 64 })) zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
1632022-08-17T20:58:25.623ZINFOSledAgent (StorageManager): Initializing Crucible
1642022-08-17T20:58:26.096ZINFOSledAgent (StorageManager): Zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 with address [fd00:1122:3344:101::8]:32345 is running
1652022-08-17T20:58:26.097ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
1662022-08-17T20:58:26.097ZINFOSledAgent (dropshot (SledAgent)): request completed local_addr = [fd00:1122:3344:101::1]:12345 method = PUT remote_addr = [fd00:1122:3344:101::1]:35538 req_id = 9eeeb205-d3f4-410c-8ae1-a85a347341e4 response_code = 204 uri = /filesystem
1672022-08-17T20:58:26.097ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 5.044761756s
1682022-08-17T20:58:26.097ZINFOSledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 4d08fc19-3d5f-4f6b-9c48-925f8eac7255, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: CockroachDb { all_addresses: [[fd00:1122:3344:101::2]:32221] }, address: [fd00:1122:3344:101::2]:32221 }
1692022-08-17T20:58:26.097ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 5.797654437s
1702022-08-17T20:58:26.097ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 127.89601ms
1712022-08-17T20:58:26.097ZINFOSledAgent (StorageManager): add_dataset: NewFilesystemRequest { zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: CockroachDb { all_addresses: [[fd00:1122:3344:101::2]:32221] }, address: [fd00:1122:3344:101::2]:32221, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
1722022-08-17T20:58:26.097ZINFOSledAgent (StorageManager): Ensuring dataset oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/cockroachdb exists
1732022-08-17T20:58:26.097ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 919.546203ms
1742022-08-17T20:58:26.140ZINFOSledAgent (StorageManager): Ensuring zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/cockroachdb is running
1752022-08-17T20:58:26.148ZINFOSledAgent (StorageManager): Zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/cockroachdb was not found
1762022-08-17T20:58:26.162ZINFOSledAgent (StorageManager): Configuring new Omicron zone: oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
1772022-08-17T20:58:26.179ZINFOSledAgent (StorageManager): Installing Omicron zone: oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
1782022-08-17T20:58:27.019ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
1792022-08-17T20:58:27.020ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.490968136s
1802022-08-17T20:58:28.265ZINFOSledAgent (StorageManager): Zone booting zone = oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
1812022-08-17T20:58:28.512ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
1822022-08-17T20:58:28.513ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 859.868621ms
1832022-08-17T20:58:29.374ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
1842022-08-17T20:58:29.374ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 989.003288ms
1852022-08-17T20:58:30.365ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
1862022-08-17T20:58:30.365ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 730.310118ms
1872022-08-17T20:58:30.840ZINFOSledAgent (StorageManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 })) zone = oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
1882022-08-17T20:58:31.407ZINFOSledAgent (StorageManager): start_zone: Loading CRDB manifest
1892022-08-17T20:58:31.600ZINFOSledAgent (StorageManager): start_zone: setting CRDB's config/listen_addr: [fd00:1122:3344:101::2]:32221
1902022-08-17T20:58:31.628ZINFOSledAgent (StorageManager): start_zone: setting CRDB's config/store
1912022-08-17T20:58:31.657ZINFOSledAgent (StorageManager): start_zone: setting CRDB's config/join_addrs
1922022-08-17T20:58:31.684ZINFOSledAgent (StorageManager): start_zone: refreshing manifest
1932022-08-17T20:58:31.711ZINFOSledAgent (StorageManager): start_zone: enabling CRDB service
1942022-08-17T20:58:31.737ZINFOSledAgent (StorageManager): start_zone: awaiting liveness of CRDB
1952022-08-17T20:58:31.743ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
1962022-08-17T20:58:31.743ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.476221319s
1972022-08-17T20:58:31.744ZWARNSledAgent (StorageManager): cockroachdb not yet alive
1982022-08-17T20:58:31.996ZWARNSledAgent (StorageManager): cockroachdb not yet alive
1992022-08-17T20:58:32.401ZWARNSledAgent (StorageManager): cockroachdb not yet alive
2002022-08-17T20:58:33.220ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
2012022-08-17T20:58:33.221ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 923.543475ms
2022022-08-17T20:58:33.725ZWARNSledAgent (StorageManager): cockroachdb not yet alive
2032022-08-17T20:58:34.146ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
2042022-08-17T20:58:34.147ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 714.773697ms
2052022-08-17T20:58:34.863ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
2062022-08-17T20:58:34.864ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 936.986186ms
2072022-08-17T20:58:34.971ZINFOSledAgent (StorageManager): CRDB is online
2082022-08-17T20:58:34.971ZINFOSledAgent (StorageManager): Formatting CRDB
2092022-08-17T20:58:35.801ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
2102022-08-17T20:58:35.802ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.083216901s
2112022-08-17T20:58:36.886ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
2122022-08-17T20:58:36.887ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.10148591s
2132022-08-17T20:58:37.989ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
2142022-08-17T20:58:37.990ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.164952307s
2152022-08-17T20:58:39.155ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
2162022-08-17T20:58:39.156ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.101061878s
2172022-08-17T20:58:40.258ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
2182022-08-17T20:58:40.259ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.372327978s
2192022-08-17T20:58:41.632ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
2202022-08-17T20:58:41.632ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 588.979295ms
2212022-08-17T20:58:42.222ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
2222022-08-17T20:58:42.223ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 598.128413ms
2232022-08-17T20:58:42.339ZINFOSledAgent (StorageManager): Formatting CRDB - Completed
2242022-08-17T20:58:42.339ZINFOSledAgent (StorageManager): Zone oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b with address [fd00:1122:3344:101::2]:32221 is running
2252022-08-17T20:58:42.339ZINFOSledAgent (dropshot (SledAgent)): request completed local_addr = [fd00:1122:3344:101::1]:12345 method = PUT remote_addr = [fd00:1122:3344:101::1]:35538 req_id = c4085914-8e51-430f-94ed-08d75f8c1014 response_code = 204 uri = /filesystem
2262022-08-17T20:58:42.339ZINFOSledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: a3505b41-a592-420b-84f2-3d76bf0e0a81, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Clickhouse, address: [fd00:1122:3344:101::5]:8123 }
2272022-08-17T20:58:42.339ZINFOSledAgent (StorageManager): add_dataset: NewFilesystemRequest { zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Clickhouse, address: [fd00:1122:3344:101::5]:8123, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
2282022-08-17T20:58:42.339ZINFOSledAgent (StorageManager): Ensuring dataset oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/clickhouse exists
2292022-08-17T20:58:42.388ZINFOSledAgent (StorageManager): Ensuring zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/clickhouse is running
2302022-08-17T20:58:42.398ZINFOSledAgent (StorageManager): Zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/clickhouse was not found
2312022-08-17T20:58:42.414ZINFOSledAgent (StorageManager): Configuring new Omicron zone: oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
2322022-08-17T20:58:42.431ZINFOSledAgent (StorageManager): Installing Omicron zone: oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
2332022-08-17T20:58:42.823ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
2342022-08-17T20:58:42.824ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 620.343174ms
2352022-08-17T20:58:43.446ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
2362022-08-17T20:58:43.446ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 688.509497ms
2372022-08-17T20:58:44.137ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
2382022-08-17T20:58:44.137ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.476753429s
2392022-08-17T20:58:44.623ZINFOSledAgent (StorageManager): Zone booting zone = oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
2402022-08-17T20:58:45.616ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
2412022-08-17T20:58:45.617ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 786.016957ms
2422022-08-17T20:58:46.250ZINFOSledAgent (StorageManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::5, prefix: 64 })) zone = oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
2432022-08-17T20:58:46.822ZINFOSledAgent (StorageManager): Initialiting Clickhouse
2442022-08-17T20:58:47.120ZINFOSledAgent (StorageManager): Zone oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b with address [fd00:1122:3344:101::5]:8123 is running
2452022-08-17T20:58:47.120ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
2462022-08-17T20:58:47.121ZINFOSledAgent (dropshot (SledAgent)): request completed local_addr = [fd00:1122:3344:101::1]:12345 method = PUT remote_addr = [fd00:1122:3344:101::1]:35538 req_id = b03e4d17-47ff-43f1-b28c-7b6d73962ec4 response_code = 204 uri = /filesystem
2472022-08-17T20:58:47.121ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 901.036969ms
2482022-08-17T20:58:47.121ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 5.360915826s
2492022-08-17T20:58:47.122ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 189.422432ms
2502022-08-17T20:58:47.122ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 4.431716735s
2512022-08-17T20:58:47.123ZINFOSledAgent (RSS): Inserting DNS record: Backend(Crucible, ffd16cad-e5d5-495e-9c59-4312a3857d91) client = DNS
2522022-08-17T20:58:47.123ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 264.749397ms
2532022-08-17T20:58:47.123ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 417.502891ms
2542022-08-17T20:58:47.123ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 193.329646ms
2552022-08-17T20:58:47.124ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 8.596784115s
2562022-08-17T20:58:47.124ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.502197423s
2572022-08-17T20:58:47.124ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 6.596661124s
2582022-08-17T20:58:47.125ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 6.416293599s
2592022-08-17T20:58:47.125ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 288.52532ms
2602022-08-17T20:58:47.126ZINFOSledAgent (RSS): Inserting DNS record: Service(Clickhouse) client = DNS
2612022-08-17T20:58:47.126ZINFOSledAgent (RSS): Inserting DNS record: Service(Cockroach) client = DNS
2622022-08-17T20:58:47.126ZINFOSledAgent (RSS): Inserting DNS record: Backend(Crucible, 09a9a25f-2602-4e2f-9630-31af9c492c3e) client = DNS
2632022-08-17T20:58:47.127ZINFOSledAgent (RSS): Inserting DNS record: Backend(Crucible, 2713b37a-3043-4ed5-aaff-f38200e45cfb) client = DNS
2642022-08-17T20:58:47.127ZINFOSledAgent (RSS): Finished setting up agents and datasets
2652022-08-17T20:58:47.131ZINFOSledAgent (RSS): sending service requests...
2662022-08-17T20:58:47.131ZINFOSledAgent (RSS): initializing sled services: [ServiceRequest { id: e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c, name: "nexus", addresses: [fd00:1122:3344:101::3], gz_addresses: [], service_type: Nexus { internal_ip: fd00:1122:3344:101::3, external_ip: 192.168.1.20 } }, ServiceRequest { id: 1da65e5b-210c-4859-a7d7-200c1e659972, name: "oximeter", addresses: [fd00:1122:3344:101::4], gz_addresses: [], service_type: Oximeter }, ServiceRequest { id: a0fe5ebc-9261-6f77-acc1-972481755789, name: "dendrite", addresses: [fd00:1122:3344:101::9], gz_addresses: [], service_type: Dendrite { asic: TofinoStub } }, ServiceRequest { id: 4b51d2cc-ccf1-49ea-b3bb-0463bb1cff8b, name: "internal-dns", addresses: [fd00:1122:3344:1::1], gz_addresses: [fd00:1122:3344:1::2], service_type: InternalDns { server_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53 } }]
2672022-08-17T20:58:47.131ZINFOSledAgent (dropshot (SledAgent)): accepted connection local_addr = [fd00:1122:3344:101::1]:12345 remote_addr = [fd00:1122:3344:101::1]:61828
2682022-08-17T20:58:47.131ZINFOSledAgent (ServiceManager): Ensuring service is initialized: ServiceRequest { id: 1da65e5b-210c-4859-a7d7-200c1e659972, name: "oximeter", addresses: [fd00:1122:3344:101::4], gz_addresses: [], service_type: Oximeter }
2692022-08-17T20:58:47.132ZINFOSledAgent (ServiceManager): Service oximeter does not yet exist
2702022-08-17T20:58:47.148ZINFOSledAgent (ServiceManager): Configuring new Omicron zone: oxz_oximeter
2712022-08-17T20:58:47.166ZINFOSledAgent (ServiceManager): Installing Omicron zone: oxz_oximeter
2722022-08-17T20:58:47.314ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 452.869246ms
2732022-08-17T20:58:47.318ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 521.146104ms
2742022-08-17T20:58:47.389ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 250.175528ms
2752022-08-17T20:58:47.415ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 582.144102ms
2762022-08-17T20:58:47.542ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 800.472421ms
2772022-08-17T20:58:47.640ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.393655769s
2782022-08-17T20:58:47.768ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 916.460152ms
2792022-08-17T20:58:47.840ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 832.022872ms
2802022-08-17T20:58:47.989ZINFOSledAgent (ServiceManager): Zone booting zone = oxz_oximeter
2812022-08-17T20:58:47.999ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 876.593613ms
2822022-08-17T20:58:48.023ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
2832022-08-17T20:58:48.024ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 596.074706ms
2842022-08-17T20:58:48.345ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.719146054s
2852022-08-17T20:58:48.621ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
2862022-08-17T20:58:48.622ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 599.622175ms
2872022-08-17T20:58:48.674ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.762387676s
2882022-08-17T20:58:48.686ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.905289499s
2892022-08-17T20:58:48.878ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 1.480041144s
2902022-08-17T20:58:49.036ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.660389238s
2912022-08-17T20:58:49.222ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
2922022-08-17T20:58:49.223ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.484687811s
2932022-08-17T20:58:49.628ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 10.769379266s
2942022-08-17T20:58:49.745ZINFOSledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::4 exists
2952022-08-17T20:58:49.745ZINFOSledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::4, prefix: 64 })) zone = oxz_oximeter
2962022-08-17T20:58:50.296ZINFOSledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::4 exists - OK
2972022-08-17T20:58:50.296ZINFOSledAgent (ServiceManager): GZ addresses: []
2982022-08-17T20:58:50.511ZINFOSledAgent (ServiceManager): Setting up oximeter service
2992022-08-17T20:58:50.613ZINFOSledAgent (ServiceManager): Ensuring service is initialized: ServiceRequest { id: e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c, name: "nexus", addresses: [fd00:1122:3344:101::3], gz_addresses: [], service_type: Nexus { internal_ip: fd00:1122:3344:101::3, external_ip: 192.168.1.20 } }
3002022-08-17T20:58:50.613ZINFOSledAgent (ServiceManager): Service nexus does not yet exist
3012022-08-17T20:58:50.633ZINFOSledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus
3022022-08-17T20:58:50.649ZINFOSledAgent (ServiceManager): Installing Omicron zone: oxz_nexus
3032022-08-17T20:58:51.702ZINFOSledAgent (ServiceManager): Zone booting zone = oxz_nexus
3042022-08-17T20:58:51.880ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
3052022-08-17T20:58:51.881ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.208346905s
3062022-08-17T20:58:51.881ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.380203515s
3072022-08-17T20:58:51.881ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 4.421353536s
3082022-08-17T20:58:51.882ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 5.772832278s
3092022-08-17T20:58:51.882ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.612661851s
3102022-08-17T20:58:51.882ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 2.71701267s
3112022-08-17T20:58:51.883ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 17.243687282s
3122022-08-17T20:58:52.484ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 10.635449452s
3132022-08-17T20:58:53.090ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
3142022-08-17T20:58:53.091ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.154766346s
3152022-08-17T20:58:53.544ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 13.432634823s
3162022-08-17T20:58:53.724ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 14.471515206s
3172022-08-17T20:58:54.105ZINFOSledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::3 exists
3182022-08-17T20:58:54.105ZINFOSledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::3, prefix: 64 })) zone = oxz_nexus
3192022-08-17T20:58:54.655ZINFOSledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::3 exists - OK
3202022-08-17T20:58:54.655ZINFOSledAgent (ServiceManager): GZ addresses: []
3212022-08-17T20:58:54.876ZINFOSledAgent (ServiceManager): Setting up Nexus service
3222022-08-17T20:58:54.877ZINFOSledAgent (ServiceManager): Adding address: Static(V4(Ipv4Network { addr: 192.168.1.20, prefix: 24 })) zone = oxz_nexus
3232022-08-17T20:58:55.380ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
3242022-08-17T20:58:55.384ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 612.703445ms
3252022-08-17T20:58:55.384ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 5.444035499s
3262022-08-17T20:58:55.384ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 6.649068817s
3272022-08-17T20:58:55.385ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 6.632216383s
3282022-08-17T20:58:55.407ZINFOSledAgent (ServiceManager): Ensuring service is initialized: ServiceRequest { id: a0fe5ebc-9261-6f77-acc1-972481755789, name: "dendrite", addresses: [fd00:1122:3344:101::9], gz_addresses: [], service_type: Dendrite { asic: TofinoStub } }
3292022-08-17T20:58:55.407ZINFOSledAgent (ServiceManager): Service dendrite does not yet exist
3302022-08-17T20:58:55.424ZINFOSledAgent (ServiceManager): Configuring new Omicron zone: oxz_dendrite
3312022-08-17T20:58:55.440ZINFOSledAgent (ServiceManager): Installing Omicron zone: oxz_dendrite
3322022-08-17T20:58:55.723ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 41.573582228s
3332022-08-17T20:58:55.998ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
3342022-08-17T20:58:55.998ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 815.22759ms
3352022-08-17T20:58:56.304ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 7.492087105s
3362022-08-17T20:58:56.815ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
3372022-08-17T20:58:56.846ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.013406456s
3382022-08-17T20:58:57.656ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 4.642613057s
3392022-08-17T20:58:57.860ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
3402022-08-17T20:58:57.861ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 502.578922ms
3412022-08-17T20:58:57.917ZINFOSledAgent (ServiceManager): Zone booting zone = oxz_dendrite
3422022-08-17T20:58:58.365ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
3432022-08-17T20:58:58.366ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.418853804s
3442022-08-17T20:58:59.785ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
3452022-08-17T20:58:59.786ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.403741363s
3462022-08-17T20:59:00.399ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 20.403475875s
3472022-08-17T20:59:00.537ZINFOSledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::9 exists
3482022-08-17T20:59:00.537ZINFOSledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::9, prefix: 64 })) zone = oxz_dendrite
3492022-08-17T20:59:01.084ZINFOSledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::9 exists - OK
3502022-08-17T20:59:01.085ZINFOSledAgent (ServiceManager): GZ addresses: []
3512022-08-17T20:59:01.298ZINFOSledAgent (ServiceManager): Setting up dendrite service
3522022-08-17T20:59:01.406ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
3532022-08-17T20:59:01.406ZINFOSledAgent (dropshot (SledAgent)): request completed local_addr = [fd00:1122:3344:101::1]:12345 method = PUT remote_addr = [fd00:1122:3344:101::1]:61828 req_id = 057181ff-f185-48f3-b9d9-1b4faa0e4b1d response_code = 204 uri = /services
3542022-08-17T20:59:01.407ZWARNSledAgent: failed to notify nexus about sled agent: no record found for name: _nexus._tcp.control-plane.oxide.internal. type: AAAA class: IN, will retry in 1.354918048s
3552022-08-17T20:59:01.407ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 9.08685705s
3562022-08-17T20:59:01.408ZINFOSledAgent (RSS): Inserting DNS record: Service(Nexus) client = DNS
3572022-08-17T20:59:01.408ZINFOSledAgent (RSS): Inserting DNS record: Service(Oximeter) client = DNS
3582022-08-17T20:59:01.408ZINFOSledAgent (RSS): Inserting DNS record: Service(Dendrite) client = DNS
3592022-08-17T20:59:01.409ZINFOSledAgent (RSS): Inserting DNS record: Service(InternalDNS) client = DNS
3602022-08-17T20:59:01.409ZINFOSledAgent (RSS): Finished setting up services
3612022-08-17T20:59:02.024ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 21.145248043s
3622022-08-17T20:59:02.040ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 16.720685243s
3632022-08-17T20:59:02.305ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 17.869728932s
3642022-08-17T20:59:02.763ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
3652022-08-17T20:59:02.768ZWARNSledAgent: failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::3]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 1.097317357s
3662022-08-17T20:59:03.802ZWARNSledAgent (StorageManager): failed to notify nexus about datasets, will retry in 16.271864098s
3672022-08-17T20:59:03.867ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
3682022-08-17T20:59:03.872ZWARNSledAgent: failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::3]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 1.007270171s
3692022-08-17T20:59:04.880ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
3702022-08-17T20:59:04.885ZWARNSledAgent: failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::3]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 907.079329ms
3712022-08-17T20:59:05.794ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7
3722022-08-17T20:59:05.799ZWARNSledAgent: failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::3]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 1.248181063s
3732022-08-17T20:59:06.990ZWARNSledAgent (StorageManager): failed to notify nexus, will retry in 22.890549373s
3742022-08-17T20:59:07.048ZINFOSledAgent: contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7