Max level shown:
12024-02-28T13:30:03.875ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 1047 local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:60804 req_id = 3937910f-a629-49a0-8a11-48eb958037fb response_code = 200 uri = /metrics/collect/2063053f-6bdd-4615-8127-179923ae9a65
22024-02-28T13:30:07.202ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 34 reason = Timeout
32024-02-28T13:30:07.202ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
42024-02-28T13:30:07.202ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
52024-02-28T13:30:07.202ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
62024-02-28T13:30:07.202ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 13 reason = Timeout
72024-02-28T13:30:07.202ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 34 reason = Timeout
82024-02-28T13:30:07.202ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
92024-02-28T13:30:07.202ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
102024-02-28T13:30:07.202ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 34 reason = Timeout
112024-02-28T13:30:07.202ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 34 reason = Timeout
122024-02-28T13:30:07.202ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 13 reason = Timeout
132024-02-28T13:30:07.202ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
142024-02-28T13:30:07.202ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 34 reason = Timeout
152024-02-28T13:30:07.203ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
162024-02-28T13:30:07.203ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
172024-02-28T13:30:07.203ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 20 reason = Timeout
182024-02-28T13:30:07.203ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0ed061a1-d644-4f36-95d9-99c559521914.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("de116760-7ce8-4051-89b5-dbe725a06843.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("de116760-7ce8-4051-89b5-dbe725a06843.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:101::8)) }], valid_until: Instant { tv_sec: 17701, tv_nsec: 789062077 } })
192024-02-28T13:30:07.203ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
202024-02-28T13:30:07.203ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
212024-02-28T13:30:07.203ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
222024-02-28T13:30:07.203ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("1f905b68-a2ec-4d9a-b15d-e713ba321080.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("842e2f1e-ea91-42d4-acc8-64d93d1c76c9.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("9e69a966-aaa5-463c-adce-1c59004ff150.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("9e69a966-aaa5-463c-adce-1c59004ff150.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:3::1)) }], valid_until: Instant { tv_sec: 17701, tv_nsec: 789393705 } })
232024-02-28T13:30:07.203ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
242024-02-28T13:30:07.204ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 56 reason = Timeout
252024-02-28T13:30:07.204ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
262024-02-28T13:30:07.204ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 20 reason = Timeout
272024-02-28T13:30:07.204ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 11 reason = Timeout
282024-02-28T13:30:07.204ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/blueprint_execution.rs:54 reason = no blueprint
292024-02-28T13:30:07.204ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
302024-02-28T13:30:07.204ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 20 reason = Timeout
312024-02-28T13:30:07.204ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 10.1\u{b5}s iteration = 11
322024-02-28T13:30:07.204ZWARNnexus (BackgroundTasks): region replacement task started background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:44
332024-02-28T13:30:07.204ZWARNnexus (BackgroundTasks): region replacement task done background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:48
342024-02-28T13:30:07.204ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 20 reason = Timeout
352024-02-28T13:30:07.204ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 7.451\u{b5}s iteration = 20
362024-02-28T13:30:07.204ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 20 reason = Timeout
372024-02-28T13:30:07.204ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
382024-02-28T13:30:07.204ZWARNnexus (BackgroundTasks): phantom disk task started background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:46
392024-02-28T13:30:07.204ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
402024-02-28T13:30:07.204ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
412024-02-28T13:30:07.204ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
422024-02-28T13:30:07.205ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
432024-02-28T13:30:07.205ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 2.436923ms iteration = 34
442024-02-28T13:30:07.205ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
452024-02-28T13:30:07.205ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
462024-02-28T13:30:07.206ZDEBGnexus (DpdClient): client request body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
472024-02-28T13:30:07.206ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
482024-02-28T13:30:07.206ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } dns_group = internal
492024-02-28T13:30:07.206ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 3.299395ms iteration = 34
502024-02-28T13:30:07.206ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
512024-02-28T13:30:07.206ZDEBGnexus: client request MgAdminClient = [fd00:1122:3344:101::2]:4676 body = None method = GET uri = http://[fd00:1122:3344:101::2]:4676/bfd/peers
522024-02-28T13:30:07.207ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
532024-02-28T13:30:07.207ZDEBGnexus (DpdClient): client response result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "bed367d5-49f5-4d3d-8eec-cf7d9ab0f2c5", "content-length": "2", "date": "Wed, 28 Feb 2024 13:30:07 GMT"} })
542024-02-28T13:30:07.207ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
552024-02-28T13:30:07.207ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
562024-02-28T13:30:07.207ZDEBGnexus: client response MgAdminClient = [fd00:1122:3344:101::2]:4676 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(4676), path: "/bfd/peers", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "857a4eab-eb3d-40ba-9058-b1b91c366695", "content-length": "2", "date": "Wed, 28 Feb 2024 13:30:07 GMT"} })
572024-02-28T13:30:07.207ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 2.754031ms iteration = 20
582024-02-28T13:30:07.207ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
592024-02-28T13:30:07.207ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
602024-02-28T13:30:07.208ZWARNnexus (BackgroundTasks): phantom disk task done background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:92
612024-02-28T13:30:07.208ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.756517ms iteration = 20
622024-02-28T13:30:07.209ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 5.636255ms iteration = 20
632024-02-28T13:30:07.210ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
642024-02-28T13:30:07.210ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
652024-02-28T13:30:07.210ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
662024-02-28T13:30:07.211ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
672024-02-28T13:30:07.212ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
682024-02-28T13:30:07.212ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
692024-02-28T13:30:07.213ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
702024-02-28T13:30:07.213ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
712024-02-28T13:30:07.213ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
722024-02-28T13:30:07.213ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
732024-02-28T13:30:07.213ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
742024-02-28T13:30:07.213ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
752024-02-28T13:30:07.213ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
762024-02-28T13:30:07.213ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
772024-02-28T13:30:07.213ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
782024-02-28T13:30:07.213ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
792024-02-28T13:30:07.213ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
802024-02-28T13:30:07.213ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
812024-02-28T13:30:07.214ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
822024-02-28T13:30:07.215ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 10.620793ms iteration = 56
832024-02-28T13:30:07.215ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
842024-02-28T13:30:07.215ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
852024-02-28T13:30:07.215ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
862024-02-28T13:30:07.215ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
872024-02-28T13:30:07.215ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
882024-02-28T13:30:07.216ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
892024-02-28T13:30:07.216ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
902024-02-28T13:30:07.216ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
912024-02-28T13:30:07.217ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
922024-02-28T13:30:07.217ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
932024-02-28T13:30:07.217ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
942024-02-28T13:30:07.217ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
952024-02-28T13:30:07.219ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
962024-02-28T13:30:07.219ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
972024-02-28T13:30:07.219ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
982024-02-28T13:30:07.219ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
992024-02-28T13:30:07.219ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
1002024-02-28T13:30:07.220ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1012024-02-28T13:30:07.220ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
1022024-02-28T13:30:07.220ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = 0201c827-5d49-4a11-bc77-0f9aeeb47a68 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
1032024-02-28T13:30:07.220ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
1042024-02-28T13:30:07.221ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
1052024-02-28T13:30:07.221ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
1062024-02-28T13:30:07.221ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
1072024-02-28T13:30:07.221ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
1082024-02-28T13:30:07.222ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1092024-02-28T13:30:07.222ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
1102024-02-28T13:30:07.222ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
1112024-02-28T13:30:07.223ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
1122024-02-28T13:30:07.223ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
1132024-02-28T13:30:07.223ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 377ad40d-9250-4e7f-a4ae-30f6b8bf3010 dns_zone_name = oxide.test found_so_far = 0 version = 2
1142024-02-28T13:30:07.223ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
1152024-02-28T13:30:07.223ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
1162024-02-28T13:30:07.224ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1172024-02-28T13:30:07.226ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
1182024-02-28T13:30:07.226ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
1192024-02-28T13:30:07.226ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
1202024-02-28T13:30:07.227ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-02-28T13:21:07.440290Z, time_modified: 2024-02-28T13:21:07.440290Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 035cfc3d-0175-4676-a7b2-5a10364daf0a, db_silo: Silo { identity: SiloIdentity { id: 035cfc3d-0175-4676-a7b2-5a10364daf0a, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-02-28T13:21:22.947127Z, time_modified: 2024-02-28T13:21:22.947127Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(4)) }, tls_certs: [TlsCertificate { digest: "eaf30322845761b046a344538c85bae76a74aa8cb20db27b417448346cdd0e53" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 035cfc3d-0175-4676-a7b2-5a10364daf0a, db_silo: Silo { identity: SiloIdentity { id: 035cfc3d-0175-4676-a7b2-5a10364daf0a, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-02-28T13:21:22.947127Z, time_modified: 2024-02-28T13:21:22.947127Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(4)) }, tls_certs: [TlsCertificate { digest: "eaf30322845761b046a344538c85bae76a74aa8cb20db27b417448346cdd0e53" }] }) }
1212024-02-28T13:30:07.227ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 24.629305ms iteration = 34
1222024-02-28T13:30:07.227ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
1232024-02-28T13:30:07.227ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
1242024-02-28T13:30:07.227ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
1252024-02-28T13:30:07.227ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
1262024-02-28T13:30:07.228ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
1272024-02-28T13:30:07.228ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = 0201c827-5d49-4a11-bc77-0f9aeeb47a68 dns_zone_name = control-plane.oxide.internal found_so_far = 49 version = 1
1282024-02-28T13:30:07.228ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
1292024-02-28T13:30:07.228ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-02-28 13:21:22.205393 UTC dns_group = internal generation = 1
1302024-02-28T13:30:07.228ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 26.161394ms iteration = 34
1312024-02-28T13:30:07.229ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 377ad40d-9250-4e7f-a4ae-30f6b8bf3010 dns_zone_name = oxide.test found_so_far = 1 version = 2
1322024-02-28T13:30:07.229ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
1332024-02-28T13:30:07.229ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-02-28 13:21:23.053802 UTC dns_group = external generation = 2
1342024-02-28T13:30:07.229ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 26.392671ms iteration = 34
1352024-02-28T13:30:07.229ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
1362024-02-28T13:30:07.229ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
1372024-02-28T13:30:07.230ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1382024-02-28T13:30:07.233ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 2c331470-b440-4c33-9aa1-35acb298a49d, lookup_type: ById(2c331470-b440-4c33-9aa1-35acb298a49d) } result = Ok(())
1392024-02-28T13:30:07.233ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
1402024-02-28T13:30:07.233ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
1412024-02-28T13:30:07.234ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 30.193468ms iteration = 20
1422024-02-28T13:30:07.269ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
1432024-02-28T13:30:07.270ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
1442024-02-28T13:30:07.328ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
1452024-02-28T13:30:07.329ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
1462024-02-28T13:30:07.329ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::8)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "2d2abd14-ccf9-4115-9689-ce10a3cb7ffd", "date": "Wed, 28 Feb 2024 13:30:07 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
1472024-02-28T13:30:07.329ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
1482024-02-28T13:30:07.331ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::9)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "58a187bb-ba0c-40cf-8c0a-4cf7f7f09e2b", "date": "Wed, 28 Feb 2024 13:30:07 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
1492024-02-28T13:30:07.331ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
1502024-02-28T13:30:07.332ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
1512024-02-28T13:30:07.332ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 129.672212ms iteration = 13
1522024-02-28T13:30:07.357ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
1532024-02-28T13:30:07.357ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:1::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "f02472f2-8c03-4b3b-b015-46faba4de193", "date": "Wed, 28 Feb 2024 13:30:07 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
1542024-02-28T13:30:07.357ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
1552024-02-28T13:30:07.359ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:3::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "95a8ba86-8a13-4cdf-86d6-75180faa81e2", "date": "Wed, 28 Feb 2024 13:30:07 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
1562024-02-28T13:30:07.359ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
1572024-02-28T13:30:07.360ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:2::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "66ed67a7-6848-4eea-9136-d718df14c004", "date": "Wed, 28 Feb 2024 13:30:07 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
1582024-02-28T13:30:07.360ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
1592024-02-28T13:30:07.361ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
1602024-02-28T13:30:07.361ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 159.00257ms iteration = 13