[ Nov 28 20:29:53 Enabled. ]
[ Nov 28 20:29:53 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ]
[ Nov 28 20:29:53 Method "start" exited with status 0. ]
[ Nov 28 20:29:53 Rereading configuration. ]
[ Nov 28 20:29:53 No 'refresh' method defined.  Treating as :true. ]
note: configured to log to "/dev/stdout"
{"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:53.445731327Z","hostname":"buskin","pid":913}
{"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:53.44587233Z","hostname":"buskin","pid":913,"path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"}
{"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:53.556871659Z","hostname":"buskin","pid":913,"interfaces":"(\"net0/linklocal\" \"net1/linklocal\")"}
{"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:53.564846007Z","hostname":"buskin","pid":913}
{"msg":"detecting (real or simulated) SP","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:53.572253211Z","hostname":"buskin","pid":913}
{"msg":"setting up bootstrap agent server","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:53.572322533Z","hostname":"buskin","pid":913}
{"msg":"Ensuring config directory exists","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:53.572387874Z","hostname":"buskin","pid":913,"path":"/var/oxide"}
{"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.074617791Z","hostname":"buskin","pid":913,"component":"BootstrapAgentServer","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.074716323Z","hostname":"buskin","pid":913,"prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, mask: 64 }"}
{"msg":"bootstrap service initializing RSS","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.074792735Z","hostname":"buskin","pid":913,"server":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"BootstrapAgent"}
{"msg":"Injecting RSS configuration: SetupServiceConfig {\n    rack_subnet: fd00:1122:3344:100::,\n    requests: [\n        HardcodedSledRequest {\n            datasets: [\n                DatasetEnsureBody {\n                    id: 09a9a25f26024e2f963031af9c492c3e,\n                    zpool_id: d462a7f7b62840fe80ff4e4189e2d62b,\n                    dataset_kind: Crucible,\n                    address: [fd00:1122:3344:101::7]:32345,\n                },\n                DatasetEnsureBody {\n                    id: 2713b37a30434ed5aafff38200e45cfb,\n                    zpool_id: e4b4dc87ab4649fba4b4d361ae214c03,\n                    dataset_kind: Crucible,\n                    address: [fd00:1122:3344:101::8]:32345,\n                },\n                DatasetEnsureBody {\n                    id: ffd16cade5d5495e9c594312a3857d91,\n                    zpool_id: f4b4dc87ab4649fba4b4d361ae214c03,\n                    dataset_kind: Crucible,\n                    address: [fd00:1122:3344:101::9]:32345,\n                },\n                DatasetEnsureBody {\n                    id: 4d08fc193d5f4f6b9c48925f8eac7255,\n                    zpool_id: d462a7f7b62840fe80ff4e4189e2d62b,\n                    dataset_kind: CockroachDb {\n                        all_addresses: [\n                            [fd00:1122:3344:101::3]:32221,\n                        ],\n                    },\n                    address: [fd00:1122:3344:101::3]:32221,\n                },\n                DatasetEnsureBody {\n                    id: a3505b41a592420b84f23d76bf0e0a81,\n                    zpool_id: d462a7f7b62840fe80ff4e4189e2d62b,\n                    dataset_kind: Clickhouse,\n                    address: [fd00:1122:3344:101::6]:8123,\n                },\n            ],\n            service_zones: [\n                ServiceZoneRequest {\n                    id: e6bff1ff24fb49dca54ec6a350cd4d6c,\n                    zone_type: Nexus,\n                    addresses: [\n                        fd00:1122:3344:101::4,\n                    ],\n                    gz_addresses: [],\n                    services: [\n                        Nexus {\n                            internal_ip: fd00:1122:3344:101::4,\n                            external_ip: 192.168.1.20,\n                        },\n                    ],\n                },\n                ServiceZoneRequest {\n                    id: 1da65e5b210c4859a7d7200c1e659972,\n                    zone_type: Oximeter,\n                    addresses: [\n                        fd00:1122:3344:101::5,\n                    ],\n                    gz_addresses: [],\n                    services: [\n                        Oximeter,\n                    ],\n                },\n            ],\n            dns_services: [],\n        },\n    ],\n    rack_secret_threshold: 1,\n    gateway: Gateway {\n        address: Some(\n            192.168.1.199,\n        ),\n        mac: MacAddr6(\n            [\n                24,\n                192,\n                77,\n                13,\n                159,\n                178,\n            ],\n        ),\n    },\n}","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.075145893Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.075204355Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"Received prefixes from ddmd","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.080698954Z","hostname":"buskin","pid":913,"component":"RSS","prefixes":"{\"fe80::8:20ff:fe84:a595\": [Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, mask: 64 }], \"fe80::8:20ff:fe1f:d856\": [Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, mask: 64 }]}"}
{"msg":"Enough peers exist to enact RSS plan","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.081097134Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.081139405Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"dns_subnets: [\n    DnsSubnet {\n        subnet: Ipv6Subnet {\n            net: Ipv6Net(\n                Ipv6Network {\n                    addr: fd00:1122:3344:1::,\n                    prefix: 64,\n                },\n            ),\n        },\n    },\n]","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.081189916Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.081226117Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.081262997Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"Plan serialized as: [\"[fdb0:18c0:4d0d:9fb2::1]:12346\".initialization_request]\nid = \"3b68b78f-7c48-48aa-a6f8-7b0b244f483f\"\nrack_id = \"13b5eda3-d280-4d3d-92d5-3e0b107addcc\"\n\n[\"[fdb0:18c0:4d0d:9fb2::1]:12346\".initialization_request.gateway]\naddress = \"192.168.1.199\"\nmac = [24, 192, 77, 13, 159, 178]\n\n[\"[fdb0:18c0:4d0d:9fb2::1]:12346\".initialization_request.subnet]\nnet = \"fd00:1122:3344:101::/64\"\n[[\"[fdb0:18c0:4d0d:9fb2::1]:12346\".services_request.dataset]]\naddress = \"[fd00:1122:3344:101::7]:32345\"\nid = \"09a9a25f-2602-4e2f-9630-31af9c492c3e\"\nzpool_id = \"d462a7f7-b628-40fe-80ff-4e4189e2d62b\"\n\n[\"[fdb0:18c0:4d0d:9fb2::1]:12346\".services_request.dataset.dataset_kind]\ntype = \"crucible\"\n\n[[\"[fdb0:18c0:4d0d:9fb2::1]:12346\".services_request.dataset]]\naddress = \"[fd00:1122:3344:101::8]:32345\"\nid = \"2713b37a-3043-4ed5-aaff-f38200e45cfb\"\nzpool_id = \"e4b4dc87-ab46-49fb-a4b4-d361ae214c03\"\n\n[\"[fdb0:18c0:4d0d:9fb2::1]:12346\".services_request.dataset.dataset_kind]\ntype = \"crucible\"\n\n[[\"[fdb0:18c0:4d0d:9fb2::1]:12346\".services_request.dataset]]\naddress = \"[fd00:1122:3344:101::9]:32345\"\nid = \"ffd16cad-e5d5-495e-9c59-4312a3857d91\"\nzpool_id = \"f4b4dc87-ab46-49fb-a4b4-d361ae214c03\"\n\n[\"[fdb0:18c0:4d0d:9fb2::1]:12346\".services_request.dataset.dataset_kind]\ntype = \"crucible\"\n\n[[\"[fdb0:18c0:4d0d:9fb2::1]:12346\".services_request.dataset]]\naddress = \"[fd00:1122:3344:101::3]:32221\"\nid = \"4d08fc19-3d5f-4f6b-9c48-925f8eac7255\"\nzpool_id = \"d462a7f7-b628-40fe-80ff-4e4189e2d62b\"\n\n[\"[fdb0:18c0:4d0d:9fb2::1]:12346\".services_request.dataset.dataset_kind]\nall_addresses = [\"[fd00:1122:3344:101::3]:32221\"]\ntype = \"cockroach_db\"\n\n[[\"[fdb0:18c0:4d0d:9fb2::1]:12346\".services_request.dataset]]\naddress = \"[fd00:1122:3344:101::6]:8123\"\nid = \"a3505b41-a592-420b-84f2-3d76bf0e0a81\"\nzpool_id = \"d462a7f7-b628-40fe-80ff-4e4189e2d62b\"\n\n[\"[fdb0:18c0:4d0d:9fb2::1]:12346\".services_request.dataset.dataset_kind]\ntype = \"clickhouse\"\n\n[[\"[fdb0:18c0:4d0d:9fb2::1]:12346\".services_request.dns_service]]\naddresses = [\"fd00:1122:3344:1::1\"]\ngz_addresses = [\"fd00:1122:3344:1::2\"]\nid = \"b8568fdf-8b3e-40d6-ae1a-8737e10bc1dc\"\nzone_type = \"internal_dns\"\n\n[[\"[fdb0:18c0:4d0d:9fb2::1]:12346\".services_request.dns_service.services]]\ndns_address = \"[fd00:1122:3344:1::1]:53\"\nserver_address = \"[fd00:1122:3344:1::1]:5353\"\ntype = \"internal_dns\"\n\n[[\"[fdb0:18c0:4d0d:9fb2::1]:12346\".services_request.service_zone]]\naddresses = [\"fd00:1122:3344:101::4\"]\ngz_addresses = []\nid = \"e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c\"\nzone_type = \"nexus\"\n\n[[\"[fdb0:18c0:4d0d:9fb2::1]:12346\".services_request.service_zone.services]]\nexternal_ip = \"192.168.1.20\"\ninternal_ip = \"fd00:1122:3344:101::4\"\ntype = \"nexus\"\n\n[[\"[fdb0:18c0:4d0d:9fb2::1]:12346\".services_request.service_zone]]\naddresses = [\"fd00:1122:3344:101::5\"]\ngz_addresses = []\nid = \"1da65e5b-210c-4859-a7d7-200c1e659972\"\nzone_type = \"oximeter\"\n\n[[\"[fdb0:18c0:4d0d:9fb2::1]:12346\".services_request.service_zone.services]]\ntype = \"oximeter\"\n","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.081518223Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"Plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.081558224Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"Skipping rack secret creation (only one sled present)","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.081594755Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.081630986Z","hostname":"buskin","pid":913,"component":"BootstrapAgentRssHandler","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"SledAgentRequest { id: 3b68b78f-7c48-48aa-a6f8-7b0b244f483f, rack_id: 13b5eda3-d280-4d3d-92d5-3e0b107addcc, gateway: Gateway { address: Some(192.168.1.199), mac: MacAddr6([24, 192, 77, 13, 159, 178]) }, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } }"}
{"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.081678667Z","hostname":"buskin","pid":913,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:49379","component":"BootstrapAgentServer"}
{"msg":"No SP available; proceeding without sprockets auth","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.081729608Z","hostname":"buskin","pid":913,"BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler"}
{"msg":"No SP available; proceeding without sprockets auth","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.08177086Z","hostname":"buskin","pid":913,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:49379","component":"BootstrapAgentServer"}
{"msg":"Loading Sled Agent: SledAgentRequest { id: 3b68b78f-7c48-48aa-a6f8-7b0b244f483f, rack_id: 13b5eda3-d280-4d3d-92d5-3e0b107addcc, gateway: Gateway { address: Some(192.168.1.199), mac: MacAddr6([24, 192, 77, 13, 159, 178]) }, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } }","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.08391046Z","hostname":"buskin","pid":913,"server":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"BootstrapAgent"}
{"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.084015993Z","hostname":"buskin","pid":913}
{"msg":"created sled agent","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.084054513Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"updating xde driver configuration file for external IP workaround","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.575862255Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent","conf_file":"\"/kernel/drv/xde.conf\""}
{"msg":"using '[AddrObject { interface: \"net0\", name: \"linklocal\" }, AddrObject { interface: \"net1\", name: \"linklocal\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.621069281Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Sled Agent upserting zpool to Storage Manager: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.75499617Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Sled Agent upserting zpool to Storage Manager: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.763280236Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Sled Agent upserting zpool to Storage Manager: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.770788733Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Storage manager processing zpool: ZpoolInfo {\n    name: \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n    size: 10200547328,\n    allocated: 127488,\n    free: 10200419840,\n    health: Online,\n}","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.782694003Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Storage manager processing zpool: ZpoolInfo {\n    name: \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n    size: 10200547328,\n    allocated: 125952,\n    free: 10200421376,\n    health: Online,\n}","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.791657435Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"No sled services found at /var/oxide/service.toml","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.798545957Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.798692631Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.798962387Z","hostname":"buskin","pid":913,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)"}
{"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.799029779Z","hostname":"buskin","pid":913,"server":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"BootstrapAgent"}
{"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.79909044Z","hostname":"buskin","pid":913,"component":"HardwareManager"}
{"msg":"Performing full hardware scan","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.799134421Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Ensuring scrimlet services (disabling services)","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.799188392Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Disabling switch zone (already complete)","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.799249134Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.799289315Z","hostname":"buskin","pid":913,"prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, mask: 64 }"}
{"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.799334066Z","hostname":"buskin","pid":913,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:49379","component":"BootstrapAgentServer"}
{"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.799379107Z","hostname":"buskin","pid":913,"component":"BootstrapAgentRssHandler","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"}
{"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.799425628Z","hostname":"buskin","pid":913,"component":"BootstrapAgentRssHandler","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"}
{"msg":"failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 359.445298ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:54.799732325Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Storage manager processing zpool: ZpoolInfo {\n    name: \"oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n    size: 10200547328,\n    allocated: 125952,\n    free: 10200421376,\n    health: Online,\n}","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.800235937Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"sending service requests...","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.807510309Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"initializing sled services: [ServiceZoneRequest { id: b8568fdf-8b3e-40d6-ae1a-8737e10bc1dc, zone_type: InternalDNS, addresses: [fd00:1122:3344:1::1], gz_addresses: [fd00:1122:3344:1::2], services: [InternalDns { server_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53 }] }]","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.807569Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.807709204Z","hostname":"buskin","pid":913,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","remote_addr":"[fd00:1122:3344:101::1]:56173"}
{"msg":"Ensuring service zone is initialized: InternalDNS","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.807906038Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"Service zone internal_dns does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.807952539Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"failed to notify nexus about datasets, will retry in 335.490899ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:54.81178276Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus, will retry in 196.271834ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:54.811827911Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 266.481293ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:54.811867951Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus, will retry in 275.160319ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:54.811942533Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus, will retry in 297.978187ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:54.811983034Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 270.08871ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:54.812037995Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.820672299Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"Installing Omicron zone: oxz_internal_dns","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:54.839272248Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"failed to notify nexus, will retry in 683.632659ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:55.009978044Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 544.544046ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:55.079372821Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 692.361989ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:55.083729164Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus, will retry in 618.828967ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:55.089026299Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus, will retry in 313.784078ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:55.111325025Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 539.797461ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:55.148697266Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:55.160668159Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 423.268178ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:55.160979566Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus, will retry in 640.051924ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:55.426695604Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:55.585750006Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 910.209008ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:55.586217407Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about datasets, will retry in 903.451155ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:55.625565325Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 941.303914ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:55.689547824Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus, will retry in 1.457379223s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:55.694702696Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus, will retry in 774.799877ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:55.709128596Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 1.243983362s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:55.777414337Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus, will retry in 1.172833731s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:56.067315384Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:56.068728958Z","hostname":"buskin","pid":913,"zone":"oxz_internal_dns","component":"ServiceManager"}
{"msg":"failed to notify nexus, will retry in 2.469135764s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:56.48598345Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:56.496905548Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 862.109587ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:56.497328808Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about datasets, will retry in 1.107745338s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:56.531603027Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 2.760520641s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:56.631839291Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 2.774480433s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:57.022883094Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus, will retry in 1.657695886s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:57.154374106Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus, will retry in 2.09509842s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:57.241627974Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:57.360632171Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: proto error: io error: No route to host (os error 148), will retry in 531.367887ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:57.360942239Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about datasets, will retry in 2.496795634s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:29:57.640485413Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Ensuring address fd00:1122:3344:1::1 exists","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:57.695144252Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:1::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:57.695189173Z","hostname":"buskin","pid":913,"zone":"oxz_internal_dns","component":"ServiceManager"}
{"msg":"Ensuring address fd00:1122:3344:1::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:58.239219346Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"GZ addresses: [\n    fd00:1122:3344:1::2,\n]","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:58.239274187Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"Ensuring GZ address fd00:1122:3344:1::2 exists","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:58.240447515Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:58.927914761Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:59.033815478Z","hostname":"buskin","pid":913,"component":"ServiceManager","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, mask: 64 }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:59.033984422Z","hostname":"buskin","pid":913,"uri":"/services","method":"PUT","req_id":"d6ed10a0-e8c1-4c89-b311-40d2d4acd4ed","remote_addr":"[fd00:1122:3344:101::1]:56173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","response_code":"204"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:59.034114516Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Adding DNS server: [fd00:1122:3344:1::1]:5353","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:59.035537299Z","hostname":"buskin","pid":913,"client":"DNS","component":"RSS"}
{"msg":"sending dataset requests...","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:59.044351077Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 09a9a25f-2602-4e2f-9630-31af9c492c3e, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::7]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:59.044396978Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:59.044512451Z","hostname":"buskin","pid":913,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","remote_addr":"[fd00:1122:3344:101::1]:51513"}
{"msg":"add_dataset: NewFilesystemRequest { zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::7]:32345, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:59.044666885Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Ensuring dataset oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible exists","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:59.044714355Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Ensuring zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible is running","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:59.087847123Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible was not found","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:59.095255998Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:59.109665708Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:29:59.126955795Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:00.595147958Z","hostname":"buskin","pid":913,"zone":"oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"StorageManager"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::7, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:02.144753419Z","hostname":"buskin","pid":913,"zone":"oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"StorageManager"}
{"msg":"Initializing Crucible","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:02.710906744Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b with address [fd00:1122:3344:101::7]:32345 is running","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:03.179324223Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:03.179784873Z","hostname":"buskin","pid":913,"uri":"/filesystem","method":"PUT","req_id":"32b66025-b8a3-448f-9f16-db104dafd8b0","remote_addr":"[fd00:1122:3344:101::1]:51513","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","response_code":"204"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 2713b37a-3043-4ed5-aaff-f38200e45cfb, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:03.179969248Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:03.180230084Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Ensuring dataset oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible exists","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:03.180277525Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Ensuring zone for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible is running","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:03.21907924Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Zone for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible was not found","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:03.226663549Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:03.241275314Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:03.258733316Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 1.414106761s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:04.03643111Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:04.720628669Z","hostname":"buskin","pid":913,"zone":"oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:05.4515462Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 957.314141ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:05.452328679Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:06.410286145Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 871.755174ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:06.410986531Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::8, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:06.966850303Z","hostname":"buskin","pid":913,"zone":"oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"StorageManager"}
{"msg":"Initializing Crucible","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:07.539388358Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 with address [fd00:1122:3344:101::8]:32345 is running","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:08.011509964Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:08.011606506Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:08.011979265Z","hostname":"buskin","pid":913,"uri":"/filesystem","method":"PUT","req_id":"6285051c-2117-4e02-a9b6-68aef7ba72a3","remote_addr":"[fd00:1122:3344:101::1]:51513","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","response_code":"204"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: ffd16cad-e5d5-495e-9c59-4312a3857d91, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:08.01216944Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 805.145153ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:08.012267512Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"add_dataset: NewFilesystemRequest { zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:08.012458106Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Ensuring dataset oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible exists","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:08.012500118Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Ensuring zone for oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible is running","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:08.056296111Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Zone for oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible was not found","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:08.064350701Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:08.081546876Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:08.099759456Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:08.818031269Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 1.0188965s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:08.81893245Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:09.595946058Z","hostname":"buskin","pid":913,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:09.838414488Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 924.955814ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:09.839475403Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:10.765843424Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 1.478327956s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:10.766491549Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::9, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:11.04466254Z","hostname":"buskin","pid":913,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"StorageManager"}
{"msg":"Initializing Crucible","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:11.615879075Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 with address [fd00:1122:3344:101::9]:32345 is running","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:12.083636808Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:12.084034027Z","hostname":"buskin","pid":913,"uri":"/filesystem","method":"PUT","req_id":"dd830234-512b-49e5-a711-97245aa181aa","remote_addr":"[fd00:1122:3344:101::1]:51513","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","response_code":"204"}
{"msg":"failed to notify nexus, will retry in 8.67405814s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:12.084211541Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 4d08fc19-3d5f-4f6b-9c48-925f8eac7255, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: CockroachDb { all_addresses: [[fd00:1122:3344:101::3]:32221] }, address: [fd00:1122:3344:101::3]:32221 }","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:12.084274083Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"failed to notify nexus about datasets, will retry in 2.584197985s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:12.084333894Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 2.141054674s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:12.084413406Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"add_dataset: NewFilesystemRequest { zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: CockroachDb { all_addresses: [[fd00:1122:3344:101::3]:32221] }, address: [fd00:1122:3344:101::3]:32221, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:12.084467737Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Ensuring dataset oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/cockroachdb exists","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:12.084534339Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Ensuring zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/cockroachdb is running","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:12.125896974Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/cockroachdb was not found","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:12.133833812Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:12.149161723Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:12.166618365Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:12.245474735Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 732.969969ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:12.246501799Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:12.980545704Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 1.210623151s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:12.981393274Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:14.193546306Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 1.435534723s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:14.19413769Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:14.627984464Z","hostname":"buskin","pid":913,"zone":"oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:15.63000278Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 1.143684066s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:15.630808409Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::3, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:16.718469675Z","hostname":"buskin","pid":913,"zone":"oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"StorageManager"}
{"msg":"start_zone: Loading CRDB manifest","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:17.283656636Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"start_zone: setting CRDB's config/listen_addr: [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:17.476895884Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"start_zone: setting CRDB's config/store","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:17.503947002Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"start_zone: setting CRDB's config/join_addrs","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:17.531948903Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"start_zone: refreshing manifest","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:17.559149244Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"start_zone: enabling CRDB service","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:17.585031415Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"start_zone: awaiting liveness of CRDB","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:17.610501276Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:17.616030916Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"cockroachdb not yet alive","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:17.616505367Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 787.512582ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:17.616959688Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"cockroachdb not yet alive","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:17.831514629Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:18.40558727Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 825.702518ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:18.406080482Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"cockroachdb not yet alive","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:18.57048781Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:19.232550936Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 652.915689ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:19.23312919Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"cockroachdb not yet alive","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:19.445599322Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:19.886898002Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 719.081597ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:19.887732091Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:20.607853817Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 1.370361023s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:20.608771319Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"CRDB is online","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:20.870954273Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:20.871022305Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:21.980901604Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 1.452614169s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:21.981677183Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:23.435753851Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 1.454391692s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:23.436473828Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:24.892547795Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 1.281351337s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:24.89319308Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:26.175266651Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 1.432427036s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:26.175987748Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:27.040764006Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Zone oxz_cockroachdb_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b with address [fd00:1122:3344:101::3]:32221 is running","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:27.040853519Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:27.041293259Z","hostname":"buskin","pid":913,"uri":"/filesystem","method":"PUT","req_id":"e076a42f-76e2-4e2c-bac8-da2f48f7efa3","remote_addr":"[fd00:1122:3344:101::1]:51513","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","response_code":"204"}
{"msg":"failed to notify nexus about datasets, will retry in 11.601582476s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:27.041411902Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 194.513245ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:27.041466933Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: a3505b41-a592-420b-84f2-3d76bf0e0a81, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Clickhouse, address: [fd00:1122:3344:101::6]:8123 }","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:27.041530295Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Clickhouse, address: [fd00:1122:3344:101::6]:8123, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:27.041674338Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Ensuring dataset oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/clickhouse exists","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:27.041724139Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Ensuring zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/clickhouse is running","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:27.095523798Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Zone for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/clickhouse was not found","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:27.104262464Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Configuring new Omicron zone: oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:27.120855026Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Installing Omicron zone: oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:27.138236836Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:27.609652156Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 833.175091ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:27.610437294Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:28.443939925Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 1.225180062s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:28.476102913Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:29.554695105Z","hostname":"buskin","pid":913,"zone":"oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:29.702122903Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 884.113101ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:29.703134327Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:30.587976789Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 1.437980741s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:30.588715076Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::6, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:31.692314138Z","hostname":"buskin","pid":913,"zone":"oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"StorageManager"}
{"msg":"Initialiting Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:32.259407484Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Zone oxz_clickhouse_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b with address [fd00:1122:3344:101::6]:8123 is running","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:32.563982108Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:32.564097141Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:32.564562162Z","hostname":"buskin","pid":913,"uri":"/filesystem","method":"PUT","req_id":"4731d7ca-cd0f-476e-a9d7-3f1a8f014028","remote_addr":"[fd00:1122:3344:101::1]:51513","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","response_code":"204"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 1.426834965s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:32.564991172Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about datasets, will retry in 343.703729ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:32.565659698Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus, will retry in 5.164469926s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:32.566036247Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Inserting DNS record: Backend(Crucible, 2713b37a-3043-4ed5-aaff-f38200e45cfb)","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:32.566269732Z","hostname":"buskin","pid":913,"client":"DNS","component":"RSS"}
{"msg":"failed to notify nexus about datasets, will retry in 200.629695ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:32.566338744Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 428.565059ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:32.566611361Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 168.085455ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:32.567544292Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 270.975707ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:32.567750837Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus, will retry in 9.66119395s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:32.568149257Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 6.632058981s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:32.568439453Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 10.335304416s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:32.56871738Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus, will retry in 3.797246005s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:32.569033107Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Inserting DNS record: Backend(Crucible, 09a9a25f-2602-4e2f-9630-31af9c492c3e)","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:32.56957569Z","hostname":"buskin","pid":913,"client":"DNS","component":"RSS"}
{"msg":"Inserting DNS record: Service(Cockroach)","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:32.569831297Z","hostname":"buskin","pid":913,"client":"DNS","component":"RSS"}
{"msg":"Inserting DNS record: Service(Clickhouse)","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:32.570007961Z","hostname":"buskin","pid":913,"client":"DNS","component":"RSS"}
{"msg":"Inserting DNS record: Backend(Crucible, ffd16cad-e5d5-495e-9c59-4312a3857d91)","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:32.570150494Z","hostname":"buskin","pid":913,"client":"DNS","component":"RSS"}
{"msg":"Finished setting up agents and datasets","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:32.570314818Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"sending service requests...","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:32.575265475Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"initializing sled services: [ServiceZoneRequest { id: e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c, zone_type: Nexus, addresses: [fd00:1122:3344:101::4], gz_addresses: [], services: [Nexus { internal_ip: fd00:1122:3344:101::4, external_ip: 192.168.1.20 }] }, ServiceZoneRequest { id: 1da65e5b-210c-4859-a7d7-200c1e659972, zone_type: Oximeter, addresses: [fd00:1122:3344:101::5], gz_addresses: [], services: [Oximeter] }, ServiceZoneRequest { id: b8568fdf-8b3e-40d6-ae1a-8737e10bc1dc, zone_type: InternalDNS, addresses: [fd00:1122:3344:1::1], gz_addresses: [fd00:1122:3344:1::2], services: [InternalDns { server_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53 }] }]","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:32.575342736Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:32.575406808Z","hostname":"buskin","pid":913,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","remote_addr":"[fd00:1122:3344:101::1]:33868"}
{"msg":"Ensuring service zone is initialized: Oximeter","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:32.575684515Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"Service zone oximeter does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:32.575745216Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"Configuring new Omicron zone: oxz_oximeter","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:32.591893177Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"Installing Omicron zone: oxz_oximeter","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:32.60897874Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"failed to notify nexus about datasets, will retry in 428.943346ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:32.736943838Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 352.777501ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:32.768707748Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 747.226442ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:32.840195714Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 291.845067ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:32.910725138Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 1.249681198s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:32.997274948Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 543.583011ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:33.12284121Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 720.78835ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:33.167448493Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 579.550139ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:33.203948203Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:33.5404161Z","hostname":"buskin","pid":913,"zone":"oxz_oximeter","component":"ServiceManager"}
{"msg":"failed to notify nexus about datasets, will retry in 1.250689749s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:33.589353925Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 2.415950978s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:33.667961259Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 1.213365708s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:33.785058611Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 1.377072895s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:33.889976086Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:33.992222377Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 798.801734ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:33.992883383Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about datasets, will retry in 2.416923378s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:34.248551724Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:34.791992963Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 574.114979ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:34.792525175Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about datasets, will retry in 1.081993668s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:34.842243218Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 3.752350669s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:34.999836635Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 4.782835542s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:35.268566834Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:35.36760789Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 1.227056464s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:35.368204104Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Ensuring address fd00:1122:3344:101::5 exists","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:35.796513997Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::5, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:35.796589279Z","hostname":"buskin","pid":913,"zone":"oxz_oximeter","component":"ServiceManager"}
{"msg":"Ensuring address fd00:1122:3344:101::5 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:36.3409948Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"GZ addresses: []","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:36.342430754Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"Setting up oximeter service","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:36.555857389Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"Ensuring service zone is initialized: Nexus","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:36.660649221Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"Service zone nexus does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:36.662070864Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:36.677192981Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 1.482127488s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:36.677852946Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about datasets, will retry in 3.207316955s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:36.678214725Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus, will retry in 9.654771396s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:36.678570623Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 4.255303022s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:36.67883216Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 4.399855223s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:36.679062165Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Configuring new Omicron zone: oxz_nexus","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:36.682549337Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"Installing Omicron zone: oxz_nexus","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:36.698950154Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"failed to notify nexus, will retry in 7.936307666s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:37.944426963Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:38.128194157Z","hostname":"buskin","pid":913,"zone":"oxz_nexus","component":"ServiceManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:38.161682997Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 1.451379296s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:38.162391074Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about datasets, will retry in 14.667099328s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:38.645452338Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 11.506167432s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:38.754248875Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 10.422577596s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:39.201658257Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:39.613998334Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 1.265659035s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:39.614737352Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about datasets, will retry in 6.871782726s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:39.887646269Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 8.239073891s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:40.053548952Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:40.881613005Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 520.8684ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:40.882469825Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about datasets, will retry in 10.734021211s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:40.936024899Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 5.518743715s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:41.080773002Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:41.404847646Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 757.223139ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:41.405649275Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:42.164754991Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 527.338373ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:42.165620111Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"Ensuring address fd00:1122:3344:101::4 exists","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:42.201866787Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::4, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:42.201941458Z","hostname":"buskin","pid":913,"zone":"oxz_nexus","component":"ServiceManager"}
{"msg":"Ensuring address fd00:1122:3344:101::4 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:42.754217536Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"GZ addresses: []","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:42.754335999Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:42.915009709Z","hostname":"buskin","pid":913,"component":"ServiceManager"}
{"msg":"Adding address: Static(V4(Ipv4Network { addr: 192.168.1.20, prefix: 24 }))","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:42.915097751Z","hostname":"buskin","pid":913,"zone":"oxz_nexus","component":"ServiceManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:43.50391974Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus, will retry in 43.376161058s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:43.507672988Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about sled agent: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: AAAA, query_class: IN }, will retry in 972.774054ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:43.50775178Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about datasets, will retry in 9.684464226s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:43.507807871Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:43.531704825Z","hostname":"buskin","pid":913,"uri":"/services","method":"PUT","req_id":"8fa680a9-e070-459e-b6fe-e057e185aaae","remote_addr":"[fd00:1122:3344:101::1]:33868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","response_code":"204"}
{"msg":"Inserting DNS record: Service(Oximeter)","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:43.533290003Z","hostname":"buskin","pid":913,"client":"DNS","component":"RSS"}
{"msg":"Inserting DNS record: Service(InternalDNS)","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:43.533638581Z","hostname":"buskin","pid":913,"client":"DNS","component":"RSS"}
{"msg":"Inserting DNS record: Service(Nexus)","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:43.533814595Z","hostname":"buskin","pid":913,"client":"DNS","component":"RSS"}
{"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:43.534001989Z","hostname":"buskin","pid":913,"component":"RSS"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:44.481716854Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 1.339707251s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:44.4879221Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:45.828881561Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 1.285005236s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:45.834996225Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus, will retry in 17.12628522s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:45.886275325Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus, will retry in 22.87957163s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:46.340100599Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 17.757926365s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:46.605184032Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 21.882761489s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:46.765654688Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:47.121737687Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 608.085686ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:47.127070412Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:47.736143149Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 575.182059ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:47.741465975Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about datasets, will retry in 21.877519545s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:48.298784061Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:48.317848601Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 1.056417525s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:48.322992282Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:49.380100217Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 949.322082ms","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:49.385522185Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about datasets, will retry in 10.707599704s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:50.266331021Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:50.336591399Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 1.090547171s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:50.341717269Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:51.433795049Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 1.060754483s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:51.438770647Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about datasets, will retry in 10.733869154s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:51.676440163Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:52.500511281Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about sled agent: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7): error trying to connect: tcp connect error: Connection refused (os error 146), will retry in 1.166583447s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:30:52.5055456Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:53.672994338Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"contacting server nexus, registering sled: fb0f7546-4d46-40ca-9d56-cbb810684ca7","v":0,"name":"SledAgent","level":30,"time":"2022-11-28T20:30:53.681727814Z","hostname":"buskin","pid":913,"sled_id":"fb0f7546-4d46-40ca-9d56-cbb810684ca7","component":"SledAgent"}
{"msg":"failed to notify nexus about datasets, will retry in 18.34026616s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:31:00.985555656Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 36.643679871s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:31:02.420831402Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 17.438215291s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:31:04.373258575Z","hostname":"buskin","pid":913,"component":"StorageManager"}
{"msg":"failed to notify nexus about datasets, will retry in 95.653425626s","v":0,"name":"SledAgent","level":40,"time":"2022-11-28T20:31:19.334931601Z","hostname":"buskin","pid":913,"component":"StorageManager"}