Max level shown:
12024-02-24T05:00:09.951ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 18 reason = Timeout
22024-02-24T05:00:09.951ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
32024-02-24T05:00:09.951ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
42024-02-24T05:00:09.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
52024-02-24T05:00:09.955ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
62024-02-24T05:00:09.955ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
72024-02-24T05:00:09.956ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
82024-02-24T05:00:09.957ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 6.557367ms iteration = 18
92024-02-24T05:00:10.202ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 180 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 0c5fe71a-0974-46ec-841e-429156fcf23a response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
102024-02-24T05:00:19.951ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 19 reason = Timeout
112024-02-24T05:00:19.951ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
122024-02-24T05:00:19.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
132024-02-24T05:00:19.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
142024-02-24T05:00:19.956ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
152024-02-24T05:00:19.956ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
162024-02-24T05:00:19.956ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
172024-02-24T05:00:19.957ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 6.048549ms iteration = 19
182024-02-24T05:00:20.203ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 164 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = d4f25303-3905-4033-8827-c66038608f76 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
192024-02-24T05:00:29.948ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 5 reason = Timeout
202024-02-24T05:00:29.948ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
212024-02-24T05:00:29.948ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
222024-02-24T05:00:29.948ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
232024-02-24T05:00:29.948ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
242024-02-24T05:00:29.949ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
252024-02-24T05:00:29.950ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
262024-02-24T05:00:29.950ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 7 reason = Timeout
272024-02-24T05:00:29.950ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 5 reason = Timeout
282024-02-24T05:00:29.950ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 5 reason = Timeout
292024-02-24T05:00:29.950ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
302024-02-24T05:00:29.950ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
312024-02-24T05:00:29.950ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 5 reason = Timeout
322024-02-24T05:00:29.950ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 5 reason = Timeout
332024-02-24T05:00:29.950ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 7 reason = Timeout
342024-02-24T05:00:29.950ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
352024-02-24T05:00:29.950ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
362024-02-24T05:00:29.950ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
372024-02-24T05:00:29.951ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
382024-02-24T05:00:29.951ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 20 reason = Timeout
392024-02-24T05:00:29.951ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 8 reason = Timeout
402024-02-24T05:00:29.951ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
412024-02-24T05:00:29.951ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 8 reason = Timeout
422024-02-24T05:00:29.951ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
432024-02-24T05:00:29.951ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
442024-02-24T05:00:29.951ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 8 reason = Timeout
452024-02-24T05:00:29.951ZWARNnexus (BackgroundTasks): phantom disk task started background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:46
462024-02-24T05:00:29.951ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
472024-02-24T05:00:29.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
482024-02-24T05:00:29.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
492024-02-24T05:00:29.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
502024-02-24T05:00:29.952ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
512024-02-24T05:00:29.952ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 8 reason = Timeout
522024-02-24T05:00:29.952ZWARNnexus (BackgroundTasks): region replacement task started background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:44
532024-02-24T05:00:29.952ZWARNnexus (BackgroundTasks): region replacement task done background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:48
542024-02-24T05:00:29.952ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 19.76\u{b5}s iteration = 8
552024-02-24T05:00:29.952ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("5012e2a2-63d7-499d-af57-9f07ffa15664.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("c91d9fea-0a9d-4406-951b-fed674d2cdbd.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("e7877af7-833e-4a24-82d3-cf26f405dfbf.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("e7877af7-833e-4a24-82d3-cf26f405dfbf.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:3::1)) }], valid_until: Instant { tv_sec: 13738, tv_nsec: 351425738 } })
562024-02-24T05:00:29.952ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("49aa1114-08ad-4373-9921-df79e027bb19.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("680000fb-bd80-416d-9626-20dd6947323d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("680000fb-bd80-416d-9626-20dd6947323d.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:101::9)) }], valid_until: Instant { tv_sec: 13738, tv_nsec: 351428487 } })
572024-02-24T05:00:29.953ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
582024-02-24T05:00:29.954ZDEBGnexus (DpdClient): client request body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
592024-02-24T05:00:29.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
602024-02-24T05:00:29.954ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
612024-02-24T05:00:29.954ZWARNnexus (BackgroundTasks): phantom disk task done background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:92
622024-02-24T05:00:29.954ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.162862ms iteration = 8
632024-02-24T05:00:29.955ZDEBGnexus (DpdClient): client response result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "0b4e18fe-871f-4bb8-b2da-6692c6f9e76c", "content-length": "1", "date": "Sat, 24 Feb 2024 05:00:29 GMT"} })
642024-02-24T05:00:29.955ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
652024-02-24T05:00:29.955ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
662024-02-24T05:00:29.955ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
672024-02-24T05:00:29.955ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
682024-02-24T05:00:29.955ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
692024-02-24T05:00:29.955ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
702024-02-24T05:00:29.956ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
712024-02-24T05:00:29.956ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
722024-02-24T05:00:29.956ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
732024-02-24T05:00:29.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
742024-02-24T05:00:29.957ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 6.167511ms iteration = 8
752024-02-24T05:00:29.957ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
762024-02-24T05:00:29.958ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } dns_group = external
772024-02-24T05:00:29.958ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 7.818443ms iteration = 5
782024-02-24T05:00:29.958ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
792024-02-24T05:00:29.959ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } dns_group = internal
802024-02-24T05:00:29.959ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 8.780036ms iteration = 5
812024-02-24T05:00:29.959ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
822024-02-24T05:00:29.959ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
832024-02-24T05:00:29.960ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
842024-02-24T05:00:29.960ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
852024-02-24T05:00:29.960ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
862024-02-24T05:00:29.960ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
872024-02-24T05:00:29.960ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
882024-02-24T05:00:29.960ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
892024-02-24T05:00:29.960ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
902024-02-24T05:00:29.960ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
912024-02-24T05:00:29.960ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
922024-02-24T05:00:29.961ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 9.966011ms iteration = 20
932024-02-24T05:00:29.961ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
942024-02-24T05:00:29.962ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
952024-02-24T05:00:29.962ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
962024-02-24T05:00:29.962ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
972024-02-24T05:00:29.964ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
982024-02-24T05:00:29.964ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
992024-02-24T05:00:29.964ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
1002024-02-24T05:00:29.964ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = ed7b83d2-8dc6-441f-a2e2-0fbc5ae652e9 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
1012024-02-24T05:00:29.964ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
1022024-02-24T05:00:29.964ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
1032024-02-24T05:00:29.964ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
1042024-02-24T05:00:29.965ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
1052024-02-24T05:00:29.965ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
1062024-02-24T05:00:29.965ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1072024-02-24T05:00:29.965ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
1082024-02-24T05:00:29.965ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
1092024-02-24T05:00:29.966ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
1102024-02-24T05:00:29.967ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1112024-02-24T05:00:29.967ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
1122024-02-24T05:00:29.967ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 6b78b467-60a6-4ddf-874b-97709602a8c7 dns_zone_name = oxide.test found_so_far = 0 version = 2
1132024-02-24T05:00:29.967ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
1142024-02-24T05:00:29.967ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
1152024-02-24T05:00:29.969ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1162024-02-24T05:00:29.970ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
1172024-02-24T05:00:29.970ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
1182024-02-24T05:00:29.970ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
1192024-02-24T05:00:29.971ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
1202024-02-24T05:00:29.971ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
1212024-02-24T05:00:29.971ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
1222024-02-24T05:00:29.972ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
1232024-02-24T05:00:29.972ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
1242024-02-24T05:00:29.973ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
1252024-02-24T05:00:29.973ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 8 reason = Timeout
1262024-02-24T05:00:29.973ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
1272024-02-24T05:00:29.973ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
1282024-02-24T05:00:29.973ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = ed7b83d2-8dc6-441f-a2e2-0fbc5ae652e9 dns_zone_name = control-plane.oxide.internal found_so_far = 49 version = 1
1292024-02-24T05:00:29.973ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
1302024-02-24T05:00:29.973ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-02-24 04:57:46.561988 UTC dns_group = internal generation = 1
1312024-02-24T05:00:29.973ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 25.621988ms iteration = 5
1322024-02-24T05:00:29.974ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 6b78b467-60a6-4ddf-874b-97709602a8c7 dns_zone_name = oxide.test found_so_far = 1 version = 2
1332024-02-24T05:00:29.974ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
1342024-02-24T05:00:29.974ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-02-24 04:57:46.984314 UTC dns_group = external generation = 2
1352024-02-24T05:00:29.974ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 24.26495ms iteration = 5
1362024-02-24T05:00:29.974ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1372024-02-24T05:00:29.976ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 5 reason = Timeout
1382024-02-24T05:00:29.976ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/blueprint_execution.rs:48 reason = no blueprint
1392024-02-24T05:00:29.976ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 10.19\u{b5}s iteration = 5
1402024-02-24T05:00:29.977ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
1412024-02-24T05:00:29.977ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
1422024-02-24T05:00:29.977ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
1432024-02-24T05:00:29.979ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
1442024-02-24T05:00:29.979ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
1452024-02-24T05:00:29.979ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-02-24T04:57:30.091889Z, time_modified: 2024-02-24T04:57:30.091889Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 14225503-e2f4-4614-85f2-4171f2c0cc28, db_silo: Silo { identity: SiloIdentity { id: 14225503-e2f4-4614-85f2-4171f2c0cc28, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-02-24T04:57:46.956873Z, time_modified: 2024-02-24T04:57:46.956873Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "1a902f55b5e588121c22f4a3e8f7791cdc445ad47d5a2a57211259fe1a5b280d" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 14225503-e2f4-4614-85f2-4171f2c0cc28, db_silo: Silo { identity: SiloIdentity { id: 14225503-e2f4-4614-85f2-4171f2c0cc28, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-02-24T04:57:46.956873Z, time_modified: 2024-02-24T04:57:46.956873Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "1a902f55b5e588121c22f4a3e8f7791cdc445ad47d5a2a57211259fe1a5b280d" }] }) }
1462024-02-24T05:00:29.979ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 29.219466ms iteration = 5
1472024-02-24T05:00:29.990ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
1482024-02-24T05:00:29.991ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
1492024-02-24T05:00:29.992ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
1502024-02-24T05:00:29.992ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
1512024-02-24T05:00:29.993ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1522024-02-24T05:00:29.996ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 318de5c0-d9bb-4803-90de-c5f48d23e50d, lookup_type: ById(318de5c0-d9bb-4803-90de-c5f48d23e50d) } result = Ok(())
1532024-02-24T05:00:29.996ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
1542024-02-24T05:00:29.996ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
1552024-02-24T05:00:29.998ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 24.648965ms iteration = 8
1562024-02-24T05:00:29.999ZDEBGnexus: client request MgAdminClient = [fd00:1122:3344:101::2]:4676 body = None method = GET uri = http://[fd00:1122:3344:101::2]:4676/bfd/peers
1572024-02-24T05:00:29.999ZDEBGnexus: client response MgAdminClient = [fd00:1122:3344:101::2]:4676 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(4676), path: "/bfd/peers", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "5c490611-2582-44c1-a142-8b03a96b400a", "content-length": "2", "date": "Sat, 24 Feb 2024 05:00:29 GMT"} })
1582024-02-24T05:00:29.999ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 47.78774ms iteration = 8
1592024-02-24T05:00:30.018ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
1602024-02-24T05:00:30.026ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
1612024-02-24T05:00:30.081ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
1622024-02-24T05:00:30.096ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
1632024-02-24T05:00:30.096ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::8)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "8f9a1b3c-6ba2-4fbd-89a7-7e8d9d19d0fa", "date": "Sat, 24 Feb 2024 05:00:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
1642024-02-24T05:00:30.096ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
1652024-02-24T05:00:30.098ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::9)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "8d2bd329-42d0-49a0-8b89-36c438eb4b37", "date": "Sat, 24 Feb 2024 05:00:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
1662024-02-24T05:00:30.098ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
1672024-02-24T05:00:30.099ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
1682024-02-24T05:00:30.099ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 148.945914ms iteration = 7
1692024-02-24T05:00:30.118ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
1702024-02-24T05:00:30.118ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:2::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "9aea2b01-658b-46e7-9f63-e02db538b2ab", "date": "Sat, 24 Feb 2024 05:00:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
1712024-02-24T05:00:30.118ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
1722024-02-24T05:00:30.119ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:3::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "8ba74300-c9c5-48a7-9b52-da3a80590abf", "date": "Sat, 24 Feb 2024 05:00:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
1732024-02-24T05:00:30.119ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
1742024-02-24T05:00:30.121ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:1::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "0aac5ad0-31e0-4dae-82fc-c916422a9153", "date": "Sat, 24 Feb 2024 05:00:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
1752024-02-24T05:00:30.121ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
1762024-02-24T05:00:30.122ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
1772024-02-24T05:00:30.122ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 171.961707ms iteration = 7
1782024-02-24T05:00:30.202ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 176 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = f8090a06-e245-46a9-9b5e-82bb7a23c4db response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
1792024-02-24T05:00:39.951ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 21 reason = Timeout
1802024-02-24T05:00:39.952ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
1812024-02-24T05:00:39.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
1822024-02-24T05:00:39.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1832024-02-24T05:00:39.956ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
1842024-02-24T05:00:39.956ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
1852024-02-24T05:00:39.956ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
1862024-02-24T05:00:39.957ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 5.872958ms iteration = 21
1872024-02-24T05:00:40.203ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 188 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 498623d1-ee8d-45e7-91d6-08e97505129d response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
1882024-02-24T05:00:49.951ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 22 reason = Timeout
1892024-02-24T05:00:49.951ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
1902024-02-24T05:00:49.951ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
1912024-02-24T05:00:49.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1922024-02-24T05:00:49.955ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
1932024-02-24T05:00:49.955ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
1942024-02-24T05:00:49.955ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
1952024-02-24T05:00:49.957ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 5.682825ms iteration = 22
1962024-02-24T05:00:50.203ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 217 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 9d686cac-8e14-49b7-9761-290a038787ee response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
1972024-02-24T05:00:59.952ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 23 reason = Timeout
1982024-02-24T05:00:59.952ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 9 reason = Timeout
1992024-02-24T05:00:59.952ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
2002024-02-24T05:00:59.952ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 9 reason = Timeout
2012024-02-24T05:00:59.952ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
2022024-02-24T05:00:59.952ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
2032024-02-24T05:00:59.952ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 9 reason = Timeout
2042024-02-24T05:00:59.952ZWARNnexus (BackgroundTasks): phantom disk task started background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:46
2052024-02-24T05:00:59.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
2062024-02-24T05:00:59.952ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 9 reason = Timeout
2072024-02-24T05:00:59.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
2082024-02-24T05:00:59.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
2092024-02-24T05:00:59.952ZWARNnexus (BackgroundTasks): region replacement task started background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:44
2102024-02-24T05:00:59.952ZWARNnexus (BackgroundTasks): region replacement task done background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:48
2112024-02-24T05:00:59.952ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 22.301\u{b5}s iteration = 9
2122024-02-24T05:00:59.954ZDEBGnexus (DpdClient): client request body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
2132024-02-24T05:00:59.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
2142024-02-24T05:00:59.954ZDEBGnexus (DpdClient): client response result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "581b9f1f-2394-4353-ac82-91f6216d493d", "content-length": "1", "date": "Sat, 24 Feb 2024 05:00:59 GMT"} })
2152024-02-24T05:00:59.954ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
2162024-02-24T05:00:59.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
2172024-02-24T05:00:59.954ZDEBGnexus: client request MgAdminClient = [fd00:1122:3344:101::2]:4676 body = None method = GET uri = http://[fd00:1122:3344:101::2]:4676/bfd/peers
2182024-02-24T05:00:59.954ZDEBGnexus: client response MgAdminClient = [fd00:1122:3344:101::2]:4676 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(4676), path: "/bfd/peers", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "688f28ae-1da1-4129-8f32-c030a94de6ed", "content-length": "2", "date": "Sat, 24 Feb 2024 05:00:59 GMT"} })
2192024-02-24T05:00:59.954ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 2.573474ms iteration = 9
2202024-02-24T05:00:59.955ZWARNnexus (BackgroundTasks): phantom disk task done background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:92
2212024-02-24T05:00:59.955ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 2.813447ms iteration = 9
2222024-02-24T05:00:59.956ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 3.727959ms iteration = 9
2232024-02-24T05:00:59.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
2242024-02-24T05:00:59.957ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
2252024-02-24T05:00:59.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
2262024-02-24T05:00:59.958ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 6.108591ms iteration = 23
2272024-02-24T05:00:59.974ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 9 reason = Timeout
2282024-02-24T05:00:59.974ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
2292024-02-24T05:00:59.974ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
2302024-02-24T05:00:59.975ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
2312024-02-24T05:00:59.978ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
2322024-02-24T05:00:59.978ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
2332024-02-24T05:00:59.978ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
2342024-02-24T05:00:59.979ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
2352024-02-24T05:00:59.979ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
2362024-02-24T05:00:59.992ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
2372024-02-24T05:00:59.993ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
2382024-02-24T05:00:59.994ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
2392024-02-24T05:00:59.994ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
2402024-02-24T05:00:59.995ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
2412024-02-24T05:00:59.997ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 318de5c0-d9bb-4803-90de-c5f48d23e50d, lookup_type: ById(318de5c0-d9bb-4803-90de-c5f48d23e50d) } result = Ok(())
2422024-02-24T05:00:59.997ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
2432024-02-24T05:00:59.997ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
2442024-02-24T05:00:59.999ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 24.969719ms iteration = 9
2452024-02-24T05:01:00.202ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 170 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = ac506335-6a62-4893-9ec1-22dabb2957c2 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
2462024-02-24T05:01:09.951ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 24 reason = Timeout
2472024-02-24T05:01:09.951ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
2482024-02-24T05:01:09.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
2492024-02-24T05:01:09.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
2502024-02-24T05:01:09.956ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
2512024-02-24T05:01:09.956ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
2522024-02-24T05:01:09.956ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
2532024-02-24T05:01:09.957ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 5.774386ms iteration = 24
2542024-02-24T05:01:10.203ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 235 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 9a81e03f-df8e-4d41-bfb0-d65c4ea78d1a response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
2552024-02-24T05:01:19.952ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 25 reason = Timeout
2562024-02-24T05:01:19.952ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
2572024-02-24T05:01:19.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
2582024-02-24T05:01:19.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
2592024-02-24T05:01:19.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
2602024-02-24T05:01:19.957ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
2612024-02-24T05:01:19.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
2622024-02-24T05:01:19.958ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 6.145111ms iteration = 25
2632024-02-24T05:01:20.203ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 546 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 24fe193f-209a-4c46-9aba-a09966605365 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
2642024-02-24T05:01:29.947ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 6 reason = Timeout
2652024-02-24T05:01:29.947ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
2662024-02-24T05:01:29.947ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
2672024-02-24T05:01:29.948ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
2682024-02-24T05:01:29.948ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
2692024-02-24T05:01:29.948ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
2702024-02-24T05:01:29.950ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
2712024-02-24T05:01:29.951ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 8 reason = Timeout
2722024-02-24T05:01:29.951ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 6 reason = Timeout
2732024-02-24T05:01:29.951ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 6 reason = Timeout
2742024-02-24T05:01:29.951ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
2752024-02-24T05:01:29.951ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
2762024-02-24T05:01:29.951ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 6 reason = Timeout
2772024-02-24T05:01:29.951ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 6 reason = Timeout
2782024-02-24T05:01:29.951ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 8 reason = Timeout
2792024-02-24T05:01:29.951ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
2802024-02-24T05:01:29.951ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
2812024-02-24T05:01:29.951ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
2822024-02-24T05:01:29.952ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
2832024-02-24T05:01:29.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
2842024-02-24T05:01:29.952ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 26 reason = Timeout
2852024-02-24T05:01:29.952ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 10 reason = Timeout
2862024-02-24T05:01:29.952ZWARNnexus (BackgroundTasks): phantom disk task started background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:46
2872024-02-24T05:01:29.952ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 10 reason = Timeout
2882024-02-24T05:01:29.952ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 10 reason = Timeout
2892024-02-24T05:01:29.952ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
2902024-02-24T05:01:29.952ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
2912024-02-24T05:01:29.952ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
2922024-02-24T05:01:29.952ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 10 reason = Timeout
2932024-02-24T05:01:29.952ZWARNnexus (BackgroundTasks): region replacement task started background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:44
2942024-02-24T05:01:29.952ZWARNnexus (BackgroundTasks): region replacement task done background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:48
2952024-02-24T05:01:29.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
2962024-02-24T05:01:29.952ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 15.031\u{b5}s iteration = 10
2972024-02-24T05:01:29.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
2982024-02-24T05:01:29.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
2992024-02-24T05:01:29.953ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
3002024-02-24T05:01:29.953ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
3012024-02-24T05:01:29.954ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
3022024-02-24T05:01:29.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
3032024-02-24T05:01:29.954ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
3042024-02-24T05:01:29.954ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
3052024-02-24T05:01:29.954ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
3062024-02-24T05:01:29.954ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
3072024-02-24T05:01:29.954ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("49aa1114-08ad-4373-9921-df79e027bb19.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("680000fb-bd80-416d-9626-20dd6947323d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("680000fb-bd80-416d-9626-20dd6947323d.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:101::9)) }], valid_until: Instant { tv_sec: 13798, tv_nsec: 353141025 } })
3082024-02-24T05:01:29.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
3092024-02-24T05:01:29.955ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("5012e2a2-63d7-499d-af57-9f07ffa15664.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("c91d9fea-0a9d-4406-951b-fed674d2cdbd.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("e7877af7-833e-4a24-82d3-cf26f405dfbf.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("e7877af7-833e-4a24-82d3-cf26f405dfbf.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:3::1)) }], valid_until: Instant { tv_sec: 13798, tv_nsec: 353328677 } })
3102024-02-24T05:01:29.955ZDEBGnexus (DpdClient): client request body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
3112024-02-24T05:01:29.955ZDEBGnexus: client request MgAdminClient = [fd00:1122:3344:101::2]:4676 body = None method = GET uri = http://[fd00:1122:3344:101::2]:4676/bfd/peers
3122024-02-24T05:01:29.956ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
3132024-02-24T05:01:29.956ZWARNnexus (BackgroundTasks): phantom disk task done background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:92
3142024-02-24T05:01:29.956ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.80807ms iteration = 10
3152024-02-24T05:01:29.956ZDEBGnexus: client response MgAdminClient = [fd00:1122:3344:101::2]:4676 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(4676), path: "/bfd/peers", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "3dc1c9da-877c-4a83-bb58-e5a378a6e1c2", "content-length": "2", "date": "Sat, 24 Feb 2024 05:01:29 GMT"} })
3162024-02-24T05:01:29.956ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 4.368238ms iteration = 10
3172024-02-24T05:01:29.957ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } dns_group = external
3182024-02-24T05:01:29.957ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
3192024-02-24T05:01:29.957ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 5.801227ms iteration = 6
3202024-02-24T05:01:29.957ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
3212024-02-24T05:01:29.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
3222024-02-24T05:01:29.958ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } dns_group = internal
3232024-02-24T05:01:29.958ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 7.050523ms iteration = 6
3242024-02-24T05:01:29.959ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
3252024-02-24T05:01:29.959ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
3262024-02-24T05:01:29.959ZDEBGnexus (DpdClient): client response result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "e321fd23-b01d-4d04-8367-e01d9c4444e9", "content-length": "1", "date": "Sat, 24 Feb 2024 05:01:29 GMT"} })
3272024-02-24T05:01:29.959ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
3282024-02-24T05:01:29.959ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
3292024-02-24T05:01:29.959ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
3302024-02-24T05:01:29.959ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
3312024-02-24T05:01:29.959ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
3322024-02-24T05:01:29.959ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
3332024-02-24T05:01:29.960ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
3342024-02-24T05:01:29.960ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
3352024-02-24T05:01:29.960ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
3362024-02-24T05:01:29.960ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
3372024-02-24T05:01:29.961ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
3382024-02-24T05:01:29.961ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
3392024-02-24T05:01:29.961ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
3402024-02-24T05:01:29.961ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
3412024-02-24T05:01:29.961ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 9.009089ms iteration = 26
3422024-02-24T05:01:29.962ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
3432024-02-24T05:01:29.962ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
3442024-02-24T05:01:29.962ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 10.168964ms iteration = 10
3452024-02-24T05:01:29.964ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
3462024-02-24T05:01:29.964ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = ed7b83d2-8dc6-441f-a2e2-0fbc5ae652e9 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
3472024-02-24T05:01:29.964ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
3482024-02-24T05:01:29.964ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
3492024-02-24T05:01:29.965ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
3502024-02-24T05:01:29.966ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
3512024-02-24T05:01:29.966ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
3522024-02-24T05:01:29.966ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
3532024-02-24T05:01:29.967ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
3542024-02-24T05:01:29.967ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
3552024-02-24T05:01:29.967ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
3562024-02-24T05:01:29.968ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
3572024-02-24T05:01:29.968ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
3582024-02-24T05:01:29.968ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
3592024-02-24T05:01:29.968ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 6b78b467-60a6-4ddf-874b-97709602a8c7 dns_zone_name = oxide.test found_so_far = 0 version = 2
3602024-02-24T05:01:29.969ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
3612024-02-24T05:01:29.969ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
3622024-02-24T05:01:29.969ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
3632024-02-24T05:01:29.969ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
3642024-02-24T05:01:29.969ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
3652024-02-24T05:01:29.974ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 10 reason = Timeout
3662024-02-24T05:01:29.974ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
3672024-02-24T05:01:29.974ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
3682024-02-24T05:01:29.975ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
3692024-02-24T05:01:29.976ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
3702024-02-24T05:01:29.977ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 6 reason = Timeout
3712024-02-24T05:01:29.977ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/blueprint_execution.rs:48 reason = no blueprint
3722024-02-24T05:01:29.977ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 16.04\u{b5}s iteration = 6
3732024-02-24T05:01:29.978ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
3742024-02-24T05:01:29.978ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
3752024-02-24T05:01:29.978ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
3762024-02-24T05:01:29.979ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
3772024-02-24T05:01:29.979ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
3782024-02-24T05:01:29.979ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
3792024-02-24T05:01:29.980ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
3802024-02-24T05:01:29.980ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = ed7b83d2-8dc6-441f-a2e2-0fbc5ae652e9 dns_zone_name = control-plane.oxide.internal found_so_far = 49 version = 1
3812024-02-24T05:01:29.980ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
3822024-02-24T05:01:29.980ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-02-24 04:57:46.561988 UTC dns_group = internal generation = 1
3832024-02-24T05:01:29.980ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 32.975475ms iteration = 6
3842024-02-24T05:01:29.981ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 6b78b467-60a6-4ddf-874b-97709602a8c7 dns_zone_name = oxide.test found_so_far = 1 version = 2
3852024-02-24T05:01:29.981ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
3862024-02-24T05:01:29.981ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-02-24 04:57:46.984314 UTC dns_group = external generation = 2
3872024-02-24T05:01:29.981ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 30.245329ms iteration = 6
3882024-02-24T05:01:29.982ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
3892024-02-24T05:01:29.983ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
3902024-02-24T05:01:29.983ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
3912024-02-24T05:01:29.984ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
3922024-02-24T05:01:29.984ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
3932024-02-24T05:01:29.988ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-02-24T04:57:30.091889Z, time_modified: 2024-02-24T04:57:30.091889Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 14225503-e2f4-4614-85f2-4171f2c0cc28, db_silo: Silo { identity: SiloIdentity { id: 14225503-e2f4-4614-85f2-4171f2c0cc28, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-02-24T04:57:46.956873Z, time_modified: 2024-02-24T04:57:46.956873Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "1a902f55b5e588121c22f4a3e8f7791cdc445ad47d5a2a57211259fe1a5b280d" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 14225503-e2f4-4614-85f2-4171f2c0cc28, db_silo: Silo { identity: SiloIdentity { id: 14225503-e2f4-4614-85f2-4171f2c0cc28, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-02-24T04:57:46.956873Z, time_modified: 2024-02-24T04:57:46.956873Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "1a902f55b5e588121c22f4a3e8f7791cdc445ad47d5a2a57211259fe1a5b280d" }] }) }
3942024-02-24T05:01:29.988ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 36.710824ms iteration = 6
3952024-02-24T05:01:29.998ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
3962024-02-24T05:01:29.998ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
3972024-02-24T05:01:30.000ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
3982024-02-24T05:01:30.000ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
3992024-02-24T05:01:30.001ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
4002024-02-24T05:01:30.005ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 318de5c0-d9bb-4803-90de-c5f48d23e50d, lookup_type: ById(318de5c0-d9bb-4803-90de-c5f48d23e50d) } result = Ok(())
4012024-02-24T05:01:30.005ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
4022024-02-24T05:01:30.005ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
4032024-02-24T05:01:30.006ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 32.64283ms iteration = 10
4042024-02-24T05:01:30.024ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
4052024-02-24T05:01:30.024ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
4062024-02-24T05:01:30.085ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
4072024-02-24T05:01:30.086ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::9)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "acd991fe-8e5c-4c14-8bbb-527adb26c0c6", "date": "Sat, 24 Feb 2024 05:01:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
4082024-02-24T05:01:30.086ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
4092024-02-24T05:01:30.087ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
4102024-02-24T05:01:30.087ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::8)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "5a8b779b-8d17-42de-a150-7ef9c353cc58", "date": "Sat, 24 Feb 2024 05:01:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
4112024-02-24T05:01:30.087ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
4122024-02-24T05:01:30.089ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
4132024-02-24T05:01:30.089ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 137.98365ms iteration = 8
4142024-02-24T05:01:30.122ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
4152024-02-24T05:01:30.122ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:1::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "f0658e50-d9cf-4b8a-be8a-909838267803", "date": "Sat, 24 Feb 2024 05:01:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
4162024-02-24T05:01:30.122ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
4172024-02-24T05:01:30.124ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:2::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "a3091df4-5fc7-4ce9-88fe-c507e0a3c876", "date": "Sat, 24 Feb 2024 05:01:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
4182024-02-24T05:01:30.124ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
4192024-02-24T05:01:30.126ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:3::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "f4cf9af2-b245-494b-abba-8002ddfee470", "date": "Sat, 24 Feb 2024 05:01:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
4202024-02-24T05:01:30.126ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
4212024-02-24T05:01:30.128ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
4222024-02-24T05:01:30.128ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 177.081775ms iteration = 8
4232024-02-24T05:01:30.203ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 232 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 6a77b51e-36d2-46c1-8494-5bd848ea8614 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
4242024-02-24T05:01:39.952ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 27 reason = Timeout
4252024-02-24T05:01:39.952ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
4262024-02-24T05:01:39.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
4272024-02-24T05:01:39.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
4282024-02-24T05:01:39.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
4292024-02-24T05:01:39.957ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
4302024-02-24T05:01:39.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
4312024-02-24T05:01:39.958ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 6.160941ms iteration = 27
4322024-02-24T05:01:40.203ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 142 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = aeb78577-c0f8-4e8d-b39a-d5fa6dea69ca response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
4332024-02-24T05:01:49.952ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 28 reason = Timeout
4342024-02-24T05:01:49.952ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
4352024-02-24T05:01:49.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
4362024-02-24T05:01:49.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
4372024-02-24T05:01:49.956ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
4382024-02-24T05:01:49.956ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
4392024-02-24T05:01:49.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
4402024-02-24T05:01:49.963ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 11.242318ms iteration = 28
4412024-02-24T05:01:50.203ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 655 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = a55843b1-943a-4e42-baf0-a1bf04c23cf2 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
4422024-02-24T05:01:59.952ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 29 reason = Timeout
4432024-02-24T05:01:59.952ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 11 reason = Timeout
4442024-02-24T05:01:59.952ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
4452024-02-24T05:01:59.952ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
4462024-02-24T05:01:59.952ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 11 reason = Timeout
4472024-02-24T05:01:59.952ZWARNnexus (BackgroundTasks): phantom disk task started background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:46
4482024-02-24T05:01:59.952ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 11 reason = Timeout
4492024-02-24T05:01:59.952ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
4502024-02-24T05:01:59.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
4512024-02-24T05:01:59.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
4522024-02-24T05:01:59.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
4532024-02-24T05:01:59.953ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 11 reason = Timeout
4542024-02-24T05:01:59.953ZWARNnexus (BackgroundTasks): region replacement task started background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:44
4552024-02-24T05:01:59.953ZWARNnexus (BackgroundTasks): region replacement task done background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:48
4562024-02-24T05:01:59.953ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 17.57\u{b5}s iteration = 11
4572024-02-24T05:01:59.954ZDEBGnexus (DpdClient): client request body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
4582024-02-24T05:01:59.954ZDEBGnexus (DpdClient): client response result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "fc420aab-defd-4e4e-ac27-d41bdda628aa", "content-length": "1", "date": "Sat, 24 Feb 2024 05:01:59 GMT"} })
4592024-02-24T05:01:59.954ZDEBGnexus: client request MgAdminClient = [fd00:1122:3344:101::2]:4676 body = None method = GET uri = http://[fd00:1122:3344:101::2]:4676/bfd/peers
4602024-02-24T05:01:59.954ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
4612024-02-24T05:01:59.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
4622024-02-24T05:01:59.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
4632024-02-24T05:01:59.954ZWARNnexus (BackgroundTasks): phantom disk task done background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:92
4642024-02-24T05:01:59.954ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 2.842408ms iteration = 11
4652024-02-24T05:01:59.955ZDEBGnexus: client response MgAdminClient = [fd00:1122:3344:101::2]:4676 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(4676), path: "/bfd/peers", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "4b6802a3-228d-454a-8ec3-1deccc451d84", "content-length": "2", "date": "Sat, 24 Feb 2024 05:01:59 GMT"} })
4662024-02-24T05:01:59.955ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 3.427015ms iteration = 11
4672024-02-24T05:01:59.956ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 4.139194ms iteration = 11
4682024-02-24T05:01:59.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
4692024-02-24T05:01:59.957ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
4702024-02-24T05:01:59.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
4712024-02-24T05:01:59.958ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 6.930591ms iteration = 29
4722024-02-24T05:01:59.974ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 11 reason = Timeout
4732024-02-24T05:01:59.974ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
4742024-02-24T05:01:59.974ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
4752024-02-24T05:01:59.975ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
4762024-02-24T05:01:59.978ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
4772024-02-24T05:01:59.978ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
4782024-02-24T05:01:59.978ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
4792024-02-24T05:01:59.979ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
4802024-02-24T05:01:59.979ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
4812024-02-24T05:01:59.990ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
4822024-02-24T05:01:59.991ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
4832024-02-24T05:01:59.992ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
4842024-02-24T05:01:59.992ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
4852024-02-24T05:01:59.993ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
4862024-02-24T05:01:59.996ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 318de5c0-d9bb-4803-90de-c5f48d23e50d, lookup_type: ById(318de5c0-d9bb-4803-90de-c5f48d23e50d) } result = Ok(())
4872024-02-24T05:01:59.996ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
4882024-02-24T05:01:59.996ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
4892024-02-24T05:01:59.997ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 23.632082ms iteration = 11
4902024-02-24T05:02:00.203ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 175 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = b2521231-a42c-4e8f-a8ca-a5b508902ecb response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
4912024-02-24T05:02:09.952ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 30 reason = Timeout
4922024-02-24T05:02:09.952ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
4932024-02-24T05:02:09.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
4942024-02-24T05:02:09.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
4952024-02-24T05:02:09.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
4962024-02-24T05:02:09.957ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
4972024-02-24T05:02:09.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
4982024-02-24T05:02:09.958ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 6.323713ms iteration = 30
4992024-02-24T05:02:10.203ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 139 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 2d63e501-e27d-4f03-be77-0daa03cef102 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
5002024-02-24T05:02:19.952ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 31 reason = Timeout
5012024-02-24T05:02:19.952ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
5022024-02-24T05:02:19.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
5032024-02-24T05:02:19.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5042024-02-24T05:02:19.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
5052024-02-24T05:02:19.957ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
5062024-02-24T05:02:19.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
5072024-02-24T05:02:19.958ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 5.683035ms iteration = 31
5082024-02-24T05:02:20.203ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 170 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 19435cea-4de9-46d7-a077-3ff7902e5871 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
5092024-02-24T05:02:29.948ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 7 reason = Timeout
5102024-02-24T05:02:29.948ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
5112024-02-24T05:02:29.948ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
5122024-02-24T05:02:29.948ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
5132024-02-24T05:02:29.949ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
5142024-02-24T05:02:29.949ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
5152024-02-24T05:02:29.950ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5162024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 9 reason = Timeout
5172024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 7 reason = Timeout
5182024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
5192024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
5202024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 7 reason = Timeout
5212024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 32 reason = Timeout
5222024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
5232024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 12 reason = Timeout
5242024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
5252024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
5262024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 12 reason = Timeout
5272024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
5282024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 7 reason = Timeout
5292024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 7 reason = Timeout
5302024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 12 reason = Timeout
5312024-02-24T05:02:29.952ZWARNnexus (BackgroundTasks): phantom disk task started background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:46
5322024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 9 reason = Timeout
5332024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
5342024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
5352024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
5362024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
5372024-02-24T05:02:29.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
5382024-02-24T05:02:29.953ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 12 reason = Timeout
5392024-02-24T05:02:29.953ZWARNnexus (BackgroundTasks): region replacement task started background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:44
5402024-02-24T05:02:29.953ZWARNnexus (BackgroundTasks): region replacement task done background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:48
5412024-02-24T05:02:29.953ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 14.71\u{b5}s iteration = 12
5422024-02-24T05:02:29.953ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
5432024-02-24T05:02:29.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
5442024-02-24T05:02:29.954ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5452024-02-24T05:02:29.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5462024-02-24T05:02:29.954ZDEBGnexus: client request MgAdminClient = [fd00:1122:3344:101::2]:4676 body = None method = GET uri = http://[fd00:1122:3344:101::2]:4676/bfd/peers
5472024-02-24T05:02:29.955ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
5482024-02-24T05:02:29.955ZDEBGnexus (DpdClient): client request body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
5492024-02-24T05:02:29.955ZWARNnexus (BackgroundTasks): phantom disk task done background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:92
5502024-02-24T05:02:29.955ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.115111ms iteration = 12
5512024-02-24T05:02:29.955ZDEBGnexus (DpdClient): client response result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "a7b360d3-3105-43ac-b69b-9bd9b970a51f", "content-length": "1", "date": "Sat, 24 Feb 2024 05:02:29 GMT"} })
5522024-02-24T05:02:29.955ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
5532024-02-24T05:02:29.955ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5542024-02-24T05:02:29.955ZDEBGnexus: client response MgAdminClient = [fd00:1122:3344:101::2]:4676 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(4676), path: "/bfd/peers", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "aab53e6d-fa4e-4b1b-91ba-c2472cddbc52", "content-length": "2", "date": "Sat, 24 Feb 2024 05:02:29 GMT"} })
5552024-02-24T05:02:29.955ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 3.650848ms iteration = 12
5562024-02-24T05:02:29.955ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
5572024-02-24T05:02:29.956ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
5582024-02-24T05:02:29.956ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
5592024-02-24T05:02:29.956ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
5602024-02-24T05:02:29.956ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
5612024-02-24T05:02:29.956ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
5622024-02-24T05:02:29.956ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("49aa1114-08ad-4373-9921-df79e027bb19.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("680000fb-bd80-416d-9626-20dd6947323d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("680000fb-bd80-416d-9626-20dd6947323d.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:101::9)) }], valid_until: Instant { tv_sec: 13858, tv_nsec: 354605450 } })
5632024-02-24T05:02:29.956ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("5012e2a2-63d7-499d-af57-9f07ffa15664.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("c91d9fea-0a9d-4406-951b-fed674d2cdbd.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("e7877af7-833e-4a24-82d3-cf26f405dfbf.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("e7877af7-833e-4a24-82d3-cf26f405dfbf.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:3::1)) }], valid_until: Instant { tv_sec: 13858, tv_nsec: 354781593 } })
5642024-02-24T05:02:29.957ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
5652024-02-24T05:02:29.957ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
5662024-02-24T05:02:29.957ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 5.392532ms iteration = 12
5672024-02-24T05:02:29.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
5682024-02-24T05:02:29.957ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5692024-02-24T05:02:29.958ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } dns_group = external
5702024-02-24T05:02:29.958ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 6.10354ms iteration = 7
5712024-02-24T05:02:29.959ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
5722024-02-24T05:02:29.959ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } dns_group = internal
5732024-02-24T05:02:29.959ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 7.200375ms iteration = 7
5742024-02-24T05:02:29.959ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
5752024-02-24T05:02:29.960ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5762024-02-24T05:02:29.960ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
5772024-02-24T05:02:29.961ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
5782024-02-24T05:02:29.961ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
5792024-02-24T05:02:29.961ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
5802024-02-24T05:02:29.961ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
5812024-02-24T05:02:29.961ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
5822024-02-24T05:02:29.961ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
5832024-02-24T05:02:29.961ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
5842024-02-24T05:02:29.961ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
5852024-02-24T05:02:29.961ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
5862024-02-24T05:02:29.961ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
5872024-02-24T05:02:29.962ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
5882024-02-24T05:02:29.962ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 10.788313ms iteration = 32
5892024-02-24T05:02:29.963ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5902024-02-24T05:02:29.963ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
5912024-02-24T05:02:29.963ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = ed7b83d2-8dc6-441f-a2e2-0fbc5ae652e9 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
5922024-02-24T05:02:29.963ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
5932024-02-24T05:02:29.963ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
5942024-02-24T05:02:29.964ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
5952024-02-24T05:02:29.964ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
5962024-02-24T05:02:29.964ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
5972024-02-24T05:02:29.964ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5982024-02-24T05:02:29.966ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
5992024-02-24T05:02:29.966ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
6002024-02-24T05:02:29.966ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
6012024-02-24T05:02:29.966ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
6022024-02-24T05:02:29.967ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
6032024-02-24T05:02:29.967ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
6042024-02-24T05:02:29.967ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
6052024-02-24T05:02:29.967ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
6062024-02-24T05:02:29.968ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
6072024-02-24T05:02:29.968ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
6082024-02-24T05:02:29.968ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 6b78b467-60a6-4ddf-874b-97709602a8c7 dns_zone_name = oxide.test found_so_far = 0 version = 2
6092024-02-24T05:02:29.968ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
6102024-02-24T05:02:29.968ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
6112024-02-24T05:02:29.970ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
6122024-02-24T05:02:29.970ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
6132024-02-24T05:02:29.970ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
6142024-02-24T05:02:29.970ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
6152024-02-24T05:02:29.970ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = ed7b83d2-8dc6-441f-a2e2-0fbc5ae652e9 dns_zone_name = control-plane.oxide.internal found_so_far = 49 version = 1
6162024-02-24T05:02:29.970ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
6172024-02-24T05:02:29.970ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-02-24 04:57:46.561988 UTC dns_group = internal generation = 1
6182024-02-24T05:02:29.970ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 22.801221ms iteration = 7
6192024-02-24T05:02:29.972ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
6202024-02-24T05:02:29.972ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
6212024-02-24T05:02:29.972ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
6222024-02-24T05:02:29.974ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 6b78b467-60a6-4ddf-874b-97709602a8c7 dns_zone_name = oxide.test found_so_far = 1 version = 2
6232024-02-24T05:02:29.974ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
6242024-02-24T05:02:29.974ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-02-24 04:57:46.984314 UTC dns_group = external generation = 2
6252024-02-24T05:02:29.974ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 21.991849ms iteration = 7
6262024-02-24T05:02:29.974ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 12 reason = Timeout
6272024-02-24T05:02:29.974ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
6282024-02-24T05:02:29.975ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
6292024-02-24T05:02:29.976ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
6302024-02-24T05:02:29.977ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 7 reason = Timeout
6312024-02-24T05:02:29.977ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/blueprint_execution.rs:48 reason = no blueprint
6322024-02-24T05:02:29.977ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 12.8\u{b5}s iteration = 7
6332024-02-24T05:02:29.977ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-02-24T04:57:30.091889Z, time_modified: 2024-02-24T04:57:30.091889Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 14225503-e2f4-4614-85f2-4171f2c0cc28, db_silo: Silo { identity: SiloIdentity { id: 14225503-e2f4-4614-85f2-4171f2c0cc28, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-02-24T04:57:46.956873Z, time_modified: 2024-02-24T04:57:46.956873Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "1a902f55b5e588121c22f4a3e8f7791cdc445ad47d5a2a57211259fe1a5b280d" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 14225503-e2f4-4614-85f2-4171f2c0cc28, db_silo: Silo { identity: SiloIdentity { id: 14225503-e2f4-4614-85f2-4171f2c0cc28, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-02-24T04:57:46.956873Z, time_modified: 2024-02-24T04:57:46.956873Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "1a902f55b5e588121c22f4a3e8f7791cdc445ad47d5a2a57211259fe1a5b280d" }] }) }
6342024-02-24T05:02:29.977ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 25.193862ms iteration = 7
6352024-02-24T05:02:29.978ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
6362024-02-24T05:02:29.978ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
6372024-02-24T05:02:29.979ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
6382024-02-24T05:02:29.980ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
6392024-02-24T05:02:29.981ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
6402024-02-24T05:02:29.991ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
6412024-02-24T05:02:29.992ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
6422024-02-24T05:02:29.993ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
6432024-02-24T05:02:29.993ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
6442024-02-24T05:02:29.994ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
6452024-02-24T05:02:29.997ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 318de5c0-d9bb-4803-90de-c5f48d23e50d, lookup_type: ById(318de5c0-d9bb-4803-90de-c5f48d23e50d) } result = Ok(())
6462024-02-24T05:02:29.997ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
6472024-02-24T05:02:29.997ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
6482024-02-24T05:02:29.998ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 23.47683ms iteration = 12
6492024-02-24T05:02:30.072ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
6502024-02-24T05:02:30.083ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
6512024-02-24T05:02:30.191ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
6522024-02-24T05:02:30.203ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 224 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 55520832-ec2e-4ff2-9e3f-e52c5c58dcee response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
6532024-02-24T05:02:30.214ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
6542024-02-24T05:02:30.215ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::9)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "0a38231c-281a-4fb0-b31f-17a407ef65c4", "date": "Sat, 24 Feb 2024 05:02:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
6552024-02-24T05:02:30.215ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
6562024-02-24T05:02:30.215ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::8)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "e3f22a80-fe25-48c5-885a-0a87b0ec26a4", "date": "Sat, 24 Feb 2024 05:02:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
6572024-02-24T05:02:30.215ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
6582024-02-24T05:02:30.217ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
6592024-02-24T05:02:30.217ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 265.082704ms iteration = 9
6602024-02-24T05:02:30.239ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
6612024-02-24T05:02:30.239ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:3::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "d6d771c2-2e8e-4fee-b6d0-52755913ec69", "date": "Sat, 24 Feb 2024 05:02:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
6622024-02-24T05:02:30.239ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
6632024-02-24T05:02:30.240ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:1::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "da7a65fd-c8bd-4d1d-94c8-bbc80ea66657", "date": "Sat, 24 Feb 2024 05:02:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
6642024-02-24T05:02:30.240ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
6652024-02-24T05:02:30.242ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:2::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "29a06d1a-bf1d-4f67-82c6-14f243ce7092", "date": "Sat, 24 Feb 2024 05:02:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
6662024-02-24T05:02:30.242ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
6672024-02-24T05:02:30.244ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
6682024-02-24T05:02:30.244ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 292.158702ms iteration = 9
6692024-02-24T05:02:39.952ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 33 reason = Timeout
6702024-02-24T05:02:39.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
6712024-02-24T05:02:39.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
6722024-02-24T05:02:39.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
6732024-02-24T05:02:39.956ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
6742024-02-24T05:02:39.957ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
6752024-02-24T05:02:39.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
6762024-02-24T05:02:39.958ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 5.447852ms iteration = 33
6772024-02-24T05:02:40.203ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 251 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 28d2ee93-409b-46f3-8251-9e3f1a98c674 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
6782024-02-24T05:02:49.952ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 34 reason = Timeout
6792024-02-24T05:02:49.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
6802024-02-24T05:02:49.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
6812024-02-24T05:02:49.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
6822024-02-24T05:02:49.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
6832024-02-24T05:02:49.957ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
6842024-02-24T05:02:49.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
6852024-02-24T05:02:49.958ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 5.667915ms iteration = 34
6862024-02-24T05:02:50.203ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 170 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 089c17aa-65e9-4dce-8acc-903e2fb28bdb response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
6872024-02-24T05:02:59.953ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 35 reason = Timeout
6882024-02-24T05:02:59.953ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 13 reason = Timeout
6892024-02-24T05:02:59.953ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
6902024-02-24T05:02:59.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
6912024-02-24T05:02:59.953ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 13 reason = Timeout
6922024-02-24T05:02:59.953ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 13 reason = Timeout
6932024-02-24T05:02:59.953ZWARNnexus (BackgroundTasks): phantom disk task started background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:46
6942024-02-24T05:02:59.953ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
6952024-02-24T05:02:59.953ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 13 reason = Timeout
6962024-02-24T05:02:59.953ZWARNnexus (BackgroundTasks): region replacement task started background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:44
6972024-02-24T05:02:59.953ZWARNnexus (BackgroundTasks): region replacement task done background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:48
6982024-02-24T05:02:59.953ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 8.481\u{b5}s iteration = 13
6992024-02-24T05:02:59.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
7002024-02-24T05:02:59.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
7012024-02-24T05:02:59.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
7022024-02-24T05:02:59.954ZDEBGnexus: client request MgAdminClient = [fd00:1122:3344:101::2]:4676 body = None method = GET uri = http://[fd00:1122:3344:101::2]:4676/bfd/peers
7032024-02-24T05:02:59.955ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7042024-02-24T05:02:59.955ZDEBGnexus (DpdClient): client request body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
7052024-02-24T05:02:59.955ZDEBGnexus (DpdClient): client response result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "69cbe39a-2dc3-4901-ba5e-817ed2cd425a", "content-length": "1", "date": "Sat, 24 Feb 2024 05:02:59 GMT"} })
7062024-02-24T05:02:59.955ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
7072024-02-24T05:02:59.955ZWARNnexus (BackgroundTasks): phantom disk task done background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:92
7082024-02-24T05:02:59.955ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 2.579074ms iteration = 13
7092024-02-24T05:02:59.955ZDEBGnexus: client response MgAdminClient = [fd00:1122:3344:101::2]:4676 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(4676), path: "/bfd/peers", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "fc39d3c1-d2fa-48f5-8fda-12e5d7ffcedf", "content-length": "2", "date": "Sat, 24 Feb 2024 05:02:59 GMT"} })
7102024-02-24T05:02:59.955ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 2.675136ms iteration = 13
7112024-02-24T05:02:59.955ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
7122024-02-24T05:02:59.957ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 3.79724ms iteration = 13
7132024-02-24T05:02:59.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
7142024-02-24T05:02:59.957ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
7152024-02-24T05:02:59.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
7162024-02-24T05:02:59.960ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 7.352437ms iteration = 35
7172024-02-24T05:02:59.974ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 13 reason = Timeout
7182024-02-24T05:02:59.974ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
7192024-02-24T05:02:59.975ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
7202024-02-24T05:02:59.976ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7212024-02-24T05:02:59.978ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
7222024-02-24T05:02:59.978ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
7232024-02-24T05:02:59.979ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
7242024-02-24T05:02:59.980ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
7252024-02-24T05:02:59.980ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
7262024-02-24T05:02:59.994ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
7272024-02-24T05:02:59.994ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
7282024-02-24T05:02:59.995ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
7292024-02-24T05:02:59.995ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
7302024-02-24T05:02:59.996ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7312024-02-24T05:02:59.999ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 318de5c0-d9bb-4803-90de-c5f48d23e50d, lookup_type: ById(318de5c0-d9bb-4803-90de-c5f48d23e50d) } result = Ok(())
7322024-02-24T05:02:59.999ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
7332024-02-24T05:02:59.999ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
7342024-02-24T05:03:00.000ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 25.594538ms iteration = 13
7352024-02-24T05:03:00.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 168 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = bf43deb5-bb6c-44ee-9df6-2ecf1f27949c response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
7362024-02-24T05:03:09.952ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 36 reason = Timeout
7372024-02-24T05:03:09.952ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
7382024-02-24T05:03:09.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
7392024-02-24T05:03:09.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7402024-02-24T05:03:09.956ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
7412024-02-24T05:03:09.956ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
7422024-02-24T05:03:09.956ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
7432024-02-24T05:03:09.958ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 5.835887ms iteration = 36
7442024-02-24T05:03:10.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 147 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 17b5acd8-e2ce-4099-a188-468734bbbbea response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
7452024-02-24T05:03:19.953ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 37 reason = Timeout
7462024-02-24T05:03:19.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
7472024-02-24T05:03:19.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
7482024-02-24T05:03:19.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7492024-02-24T05:03:19.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
7502024-02-24T05:03:19.957ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
7512024-02-24T05:03:19.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
7522024-02-24T05:03:19.958ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 5.404671ms iteration = 37
7532024-02-24T05:03:20.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 164 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = af895a1e-9067-4be8-8f35-852c691b266b response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
7542024-02-24T05:03:29.949ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 8 reason = Timeout
7552024-02-24T05:03:29.949ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
7562024-02-24T05:03:29.949ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
7572024-02-24T05:03:29.949ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
7582024-02-24T05:03:29.949ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
7592024-02-24T05:03:29.950ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
7602024-02-24T05:03:29.951ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 10 reason = Timeout
7612024-02-24T05:03:29.951ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 8 reason = Timeout
7622024-02-24T05:03:29.951ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
7632024-02-24T05:03:29.951ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
7642024-02-24T05:03:29.951ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 8 reason = Timeout
7652024-02-24T05:03:29.951ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
7662024-02-24T05:03:29.951ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 10 reason = Timeout
7672024-02-24T05:03:29.951ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7682024-02-24T05:03:29.951ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 8 reason = Timeout
7692024-02-24T05:03:29.951ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 8 reason = Timeout
7702024-02-24T05:03:29.951ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
7712024-02-24T05:03:29.951ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
7722024-02-24T05:03:29.951ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
7732024-02-24T05:03:29.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
7742024-02-24T05:03:29.952ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 38 reason = Timeout
7752024-02-24T05:03:29.952ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 14 reason = Timeout
7762024-02-24T05:03:29.952ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
7772024-02-24T05:03:29.952ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
7782024-02-24T05:03:29.952ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 14 reason = Timeout
7792024-02-24T05:03:29.952ZWARNnexus (BackgroundTasks): phantom disk task started background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:46
7802024-02-24T05:03:29.952ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 14 reason = Timeout
7812024-02-24T05:03:29.952ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
7822024-02-24T05:03:29.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
7832024-02-24T05:03:29.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
7842024-02-24T05:03:29.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
7852024-02-24T05:03:29.953ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("49aa1114-08ad-4373-9921-df79e027bb19.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("680000fb-bd80-416d-9626-20dd6947323d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("680000fb-bd80-416d-9626-20dd6947323d.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:101::9)) }], valid_until: Instant { tv_sec: 13918, tv_nsec: 350839355 } })
7862024-02-24T05:03:29.953ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("5012e2a2-63d7-499d-af57-9f07ffa15664.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("c91d9fea-0a9d-4406-951b-fed674d2cdbd.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("e7877af7-833e-4a24-82d3-cf26f405dfbf.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("e7877af7-833e-4a24-82d3-cf26f405dfbf.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:3::1)) }], valid_until: Instant { tv_sec: 13918, tv_nsec: 350854845 } })
7872024-02-24T05:03:29.953ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7882024-02-24T05:03:29.953ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 14 reason = Timeout
7892024-02-24T05:03:29.953ZWARNnexus (BackgroundTasks): region replacement task started background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:44
7902024-02-24T05:03:29.953ZWARNnexus (BackgroundTasks): region replacement task done background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:48
7912024-02-24T05:03:29.953ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 11.629\u{b5}s iteration = 14
7922024-02-24T05:03:29.953ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7932024-02-24T05:03:29.954ZDEBGnexus: client request MgAdminClient = [fd00:1122:3344:101::2]:4676 body = None method = GET uri = http://[fd00:1122:3344:101::2]:4676/bfd/peers
7942024-02-24T05:03:29.955ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7952024-02-24T05:03:29.955ZDEBGnexus (DpdClient): client request body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
7962024-02-24T05:03:29.955ZDEBGnexus: client response MgAdminClient = [fd00:1122:3344:101::2]:4676 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(4676), path: "/bfd/peers", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "52986cbc-4a5d-4ce8-8551-29620232431c", "content-length": "2", "date": "Sat, 24 Feb 2024 05:03:29 GMT"} })
7972024-02-24T05:03:29.955ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 2.926349ms iteration = 14
7982024-02-24T05:03:29.955ZWARNnexus (BackgroundTasks): phantom disk task done background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:92
7992024-02-24T05:03:29.955ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.101221ms iteration = 14
8002024-02-24T05:03:29.957ZDEBGnexus (DpdClient): client response result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "94a523d9-c630-45d7-a5fc-9377dc398fe4", "content-length": "1", "date": "Sat, 24 Feb 2024 05:03:29 GMT"} })
8012024-02-24T05:03:29.957ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
8022024-02-24T05:03:29.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
8032024-02-24T05:03:29.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
8042024-02-24T05:03:29.957ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
8052024-02-24T05:03:29.957ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
8062024-02-24T05:03:29.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
8072024-02-24T05:03:29.958ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } dns_group = external
8082024-02-24T05:03:29.958ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 6.549027ms iteration = 8
8092024-02-24T05:03:29.958ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
8102024-02-24T05:03:29.958ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
8112024-02-24T05:03:29.958ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
8122024-02-24T05:03:29.958ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
8132024-02-24T05:03:29.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
8142024-02-24T05:03:29.958ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 6.096241ms iteration = 14
8152024-02-24T05:03:29.959ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
8162024-02-24T05:03:29.959ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } dns_group = internal
8172024-02-24T05:03:29.959ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 7.686832ms iteration = 8
8182024-02-24T05:03:29.959ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
8192024-02-24T05:03:29.959ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
8202024-02-24T05:03:29.959ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
8212024-02-24T05:03:29.959ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
8222024-02-24T05:03:29.960ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
8232024-02-24T05:03:29.960ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8242024-02-24T05:03:29.960ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
8252024-02-24T05:03:29.960ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
8262024-02-24T05:03:29.960ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
8272024-02-24T05:03:29.960ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
8282024-02-24T05:03:29.960ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
8292024-02-24T05:03:29.960ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8302024-02-24T05:03:29.961ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 8.281939ms iteration = 38
8312024-02-24T05:03:29.961ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8322024-02-24T05:03:29.963ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
8332024-02-24T05:03:29.963ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
8342024-02-24T05:03:29.963ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
8352024-02-24T05:03:29.965ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
8362024-02-24T05:03:29.965ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = ed7b83d2-8dc6-441f-a2e2-0fbc5ae652e9 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
8372024-02-24T05:03:29.965ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
8382024-02-24T05:03:29.965ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
8392024-02-24T05:03:29.965ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
8402024-02-24T05:03:29.965ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
8412024-02-24T05:03:29.965ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
8422024-02-24T05:03:29.966ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
8432024-02-24T05:03:29.966ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
8442024-02-24T05:03:29.966ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
8452024-02-24T05:03:29.966ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8462024-02-24T05:03:29.967ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
8472024-02-24T05:03:29.968ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
8482024-02-24T05:03:29.968ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 6b78b467-60a6-4ddf-874b-97709602a8c7 dns_zone_name = oxide.test found_so_far = 0 version = 2
8492024-02-24T05:03:29.968ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
8502024-02-24T05:03:29.968ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
8512024-02-24T05:03:29.968ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
8522024-02-24T05:03:29.970ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
8532024-02-24T05:03:29.970ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
8542024-02-24T05:03:29.970ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
8552024-02-24T05:03:29.975ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 14 reason = Timeout
8562024-02-24T05:03:29.975ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8572024-02-24T05:03:29.975ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8582024-02-24T05:03:29.975ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
8592024-02-24T05:03:29.975ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
8602024-02-24T05:03:29.978ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 8 reason = Timeout
8612024-02-24T05:03:29.978ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/blueprint_execution.rs:48 reason = no blueprint
8622024-02-24T05:03:29.978ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 11\u{b5}s iteration = 8
8632024-02-24T05:03:29.978ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
8642024-02-24T05:03:29.978ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
8652024-02-24T05:03:29.979ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
8662024-02-24T05:03:29.979ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = ed7b83d2-8dc6-441f-a2e2-0fbc5ae652e9 dns_zone_name = control-plane.oxide.internal found_so_far = 49 version = 1
8672024-02-24T05:03:29.979ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
8682024-02-24T05:03:29.979ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-02-24 04:57:46.561988 UTC dns_group = internal generation = 1
8692024-02-24T05:03:29.979ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 30.474071ms iteration = 8
8702024-02-24T05:03:29.979ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
8712024-02-24T05:03:29.979ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
8722024-02-24T05:03:29.980ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
8732024-02-24T05:03:29.980ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8742024-02-24T05:03:29.981ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 6b78b467-60a6-4ddf-874b-97709602a8c7 dns_zone_name = oxide.test found_so_far = 1 version = 2
8752024-02-24T05:03:29.981ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
8762024-02-24T05:03:29.981ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-02-24 04:57:46.984314 UTC dns_group = external generation = 2
8772024-02-24T05:03:29.981ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 30.025126ms iteration = 8
8782024-02-24T05:03:29.983ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
8792024-02-24T05:03:29.983ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
8802024-02-24T05:03:29.983ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
8812024-02-24T05:03:29.984ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-02-24T04:57:30.091889Z, time_modified: 2024-02-24T04:57:30.091889Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 14225503-e2f4-4614-85f2-4171f2c0cc28, db_silo: Silo { identity: SiloIdentity { id: 14225503-e2f4-4614-85f2-4171f2c0cc28, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-02-24T04:57:46.956873Z, time_modified: 2024-02-24T04:57:46.956873Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "1a902f55b5e588121c22f4a3e8f7791cdc445ad47d5a2a57211259fe1a5b280d" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 14225503-e2f4-4614-85f2-4171f2c0cc28, db_silo: Silo { identity: SiloIdentity { id: 14225503-e2f4-4614-85f2-4171f2c0cc28, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-02-24T04:57:46.956873Z, time_modified: 2024-02-24T04:57:46.956873Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "1a902f55b5e588121c22f4a3e8f7791cdc445ad47d5a2a57211259fe1a5b280d" }] }) }
8822024-02-24T05:03:29.985ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 33.541493ms iteration = 8
8832024-02-24T05:03:29.985ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
8842024-02-24T05:03:29.985ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
8852024-02-24T05:03:29.997ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
8862024-02-24T05:03:29.997ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
8872024-02-24T05:03:29.998ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
8882024-02-24T05:03:29.999ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
8892024-02-24T05:03:30.000ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8902024-02-24T05:03:30.002ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 318de5c0-d9bb-4803-90de-c5f48d23e50d, lookup_type: ById(318de5c0-d9bb-4803-90de-c5f48d23e50d) } result = Ok(())
8912024-02-24T05:03:30.002ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
8922024-02-24T05:03:30.002ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
8932024-02-24T05:03:30.003ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 28.641198ms iteration = 14
8942024-02-24T05:03:30.021ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
8952024-02-24T05:03:30.023ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
8962024-02-24T05:03:30.085ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
8972024-02-24T05:03:30.085ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::9)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "1a51a6b6-b0ee-4984-b402-8182b936e331", "date": "Sat, 24 Feb 2024 05:03:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
8982024-02-24T05:03:30.085ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
8992024-02-24T05:03:30.086ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::8)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "1e42ea0b-af6b-4fdd-9035-02737f151aef", "date": "Sat, 24 Feb 2024 05:03:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
9002024-02-24T05:03:30.086ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
9012024-02-24T05:03:30.088ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
9022024-02-24T05:03:30.088ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 136.53038ms iteration = 10
9032024-02-24T05:03:30.088ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
9042024-02-24T05:03:30.118ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
9052024-02-24T05:03:30.119ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:3::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "cbb4240c-6953-4dd3-8c8d-b11998fa89b3", "date": "Sat, 24 Feb 2024 05:03:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
9062024-02-24T05:03:30.119ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
9072024-02-24T05:03:30.119ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:2::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "38411767-20f3-4ac0-a354-36cafcf3f170", "date": "Sat, 24 Feb 2024 05:03:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
9082024-02-24T05:03:30.119ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
9092024-02-24T05:03:30.121ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:1::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "ad392dbd-3bc4-4a24-ba92-cd67f73e4c98", "date": "Sat, 24 Feb 2024 05:03:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
9102024-02-24T05:03:30.121ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
9112024-02-24T05:03:30.123ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
9122024-02-24T05:03:30.123ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 172.021088ms iteration = 10
9132024-02-24T05:03:30.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 155 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = fb417c31-00bf-4c9b-addc-1037cd8ceb68 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
9142024-02-24T05:03:39.953ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 39 reason = Timeout
9152024-02-24T05:03:39.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
9162024-02-24T05:03:39.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
9172024-02-24T05:03:39.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9182024-02-24T05:03:39.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
9192024-02-24T05:03:39.957ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
9202024-02-24T05:03:39.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
9212024-02-24T05:03:39.958ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 5.550493ms iteration = 39
9222024-02-24T05:03:40.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 181 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 27b2d739-3c95-48da-97ca-a07bae703fce response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
9232024-02-24T05:03:49.952ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 40 reason = Timeout
9242024-02-24T05:03:49.952ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
9252024-02-24T05:03:49.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
9262024-02-24T05:03:49.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9272024-02-24T05:03:49.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
9282024-02-24T05:03:49.957ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
9292024-02-24T05:03:49.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
9302024-02-24T05:03:49.958ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 5.985019ms iteration = 40
9312024-02-24T05:03:50.203ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 160 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = c1d7b9a2-7505-4650-a9d5-33d50c9e70fa response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
9322024-02-24T05:03:59.953ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 41 reason = Timeout
9332024-02-24T05:03:59.953ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 15 reason = Timeout
9342024-02-24T05:03:59.953ZWARNnexus (BackgroundTasks): phantom disk task started background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:46
9352024-02-24T05:03:59.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
9362024-02-24T05:03:59.953ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 15 reason = Timeout
9372024-02-24T05:03:59.953ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 15 reason = Timeout
9382024-02-24T05:03:59.953ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
9392024-02-24T05:03:59.953ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
9402024-02-24T05:03:59.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
9412024-02-24T05:03:59.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
9422024-02-24T05:03:59.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
9432024-02-24T05:03:59.954ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 15 reason = Timeout
9442024-02-24T05:03:59.954ZWARNnexus (BackgroundTasks): region replacement task started background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:44
9452024-02-24T05:03:59.954ZWARNnexus (BackgroundTasks): region replacement task done background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:48
9462024-02-24T05:03:59.954ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 18.87\u{b5}s iteration = 15
9472024-02-24T05:03:59.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9482024-02-24T05:03:59.955ZDEBGnexus (DpdClient): client request body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
9492024-02-24T05:03:59.955ZDEBGnexus: client request MgAdminClient = [fd00:1122:3344:101::2]:4676 body = None method = GET uri = http://[fd00:1122:3344:101::2]:4676/bfd/peers
9502024-02-24T05:03:59.955ZWARNnexus (BackgroundTasks): phantom disk task done background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:92
9512024-02-24T05:03:59.955ZDEBGnexus (DpdClient): client response result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "7c926fd3-5689-4959-af78-34d1a1743c32", "content-length": "1", "date": "Sat, 24 Feb 2024 05:03:59 GMT"} })
9522024-02-24T05:03:59.955ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 2.29136ms iteration = 15
9532024-02-24T05:03:59.955ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
9542024-02-24T05:03:59.955ZDEBGnexus: client response MgAdminClient = [fd00:1122:3344:101::2]:4676 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(4676), path: "/bfd/peers", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "4a62fe2b-c178-4809-a739-70481119eddd", "content-length": "2", "date": "Sat, 24 Feb 2024 05:03:59 GMT"} })
9552024-02-24T05:03:59.955ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 2.307591ms iteration = 15
9562024-02-24T05:03:59.955ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
9572024-02-24T05:03:59.956ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 3.75879ms iteration = 15
9582024-02-24T05:03:59.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
9592024-02-24T05:03:59.957ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
9602024-02-24T05:03:59.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
9612024-02-24T05:03:59.959ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 5.932188ms iteration = 41
9622024-02-24T05:03:59.975ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 15 reason = Timeout
9632024-02-24T05:03:59.975ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
9642024-02-24T05:03:59.975ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
9652024-02-24T05:03:59.976ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9662024-02-24T05:03:59.979ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
9672024-02-24T05:03:59.979ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
9682024-02-24T05:03:59.979ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
9692024-02-24T05:03:59.980ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
9702024-02-24T05:03:59.981ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
9712024-02-24T05:03:59.994ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
9722024-02-24T05:03:59.994ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
9732024-02-24T05:03:59.995ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
9742024-02-24T05:03:59.995ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
9752024-02-24T05:03:59.996ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9762024-02-24T05:03:59.999ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 318de5c0-d9bb-4803-90de-c5f48d23e50d, lookup_type: ById(318de5c0-d9bb-4803-90de-c5f48d23e50d) } result = Ok(())
9772024-02-24T05:03:59.999ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
9782024-02-24T05:03:59.999ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
9792024-02-24T05:04:00.000ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 25.421595ms iteration = 15
9802024-02-24T05:04:00.205ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 1067 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 8ff43067-0f6b-4efe-9782-2ddebace50c3 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
9812024-02-24T05:04:09.952ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 42 reason = Timeout
9822024-02-24T05:04:09.952ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
9832024-02-24T05:04:09.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
9842024-02-24T05:04:09.959ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9852024-02-24T05:04:09.962ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
9862024-02-24T05:04:09.962ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
9872024-02-24T05:04:09.962ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
9882024-02-24T05:04:09.964ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 12.14379ms iteration = 42
9892024-02-24T05:04:10.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 252 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = b4bf7685-7027-44a0-a090-7f4b79fbb555 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
9902024-02-24T05:04:17.226ZINFOnexus (dropshot_internal): accepted connection file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765 local_addr = [fd00:1122:3344:101::b]:12221 remote_addr = [fd00:1122:3344:101::1:0]:62593
9912024-02-24T05:04:17.264ZINFOnexus: registered oximeter collector client file = nexus/src/app/oximeter.rs:374 id = c5f09c20-98cf-43c3-9a57-f953993a48b3
9922024-02-24T05:04:17.285ZDEBGnexus: client request body = Some(Body) method = POST oximeter-collector = c5f09c20-98cf-43c3-9a57-f953993a48b3 uri = http://[fd00:1122:3344:101::d]:12223/producers
9932024-02-24T05:04:17.286ZDEBGnexus: client response oximeter-collector = c5f09c20-98cf-43c3-9a57-f953993a48b3 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::d)), port: Some(12223), path: "/producers", query: None, fragment: None }, status: 204, headers: {"x-request-id": "e1654b87-9000-4b66-871c-b53b03ffb3b6", "date": "Sat, 24 Feb 2024 05:04:17 GMT"} })
9942024-02-24T05:04:17.286ZINFOnexus: assigned collector to new producer collector_id = c5f09c20-98cf-43c3-9a57-f953993a48b3 file = nexus/src/app/oximeter.rs:193 producer_id = 9875b9bb-c32e-47f6-ba67-0fe3403ccbf4
9952024-02-24T05:04:17.287ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 60620 local_addr = [fd00:1122:3344:101::b]:12221 method = POST remote_addr = [fd00:1122:3344:101::1:0]:62593 req_id = 6fd9bd00-ec8d-49fe-a962-c35845e3a205 response_code = 204 uri = /metrics/producers
9962024-02-24T05:04:19.953ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 43 reason = Timeout
9972024-02-24T05:04:19.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
9982024-02-24T05:04:19.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
9992024-02-24T05:04:19.955ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10002024-02-24T05:04:19.958ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
10012024-02-24T05:04:19.958ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
10022024-02-24T05:04:19.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
10032024-02-24T05:04:19.960ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 6.617727ms iteration = 43
10042024-02-24T05:04:20.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 173 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 1afa1c35-3fbd-4140-8982-e27bb7897e8c response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
10052024-02-24T05:04:29.949ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 9 reason = Timeout
10062024-02-24T05:04:29.949ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
10072024-02-24T05:04:29.949ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
10082024-02-24T05:04:29.950ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
10092024-02-24T05:04:29.950ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
10102024-02-24T05:04:29.950ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
10112024-02-24T05:04:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 11 reason = Timeout
10122024-02-24T05:04:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 9 reason = Timeout
10132024-02-24T05:04:29.952ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
10142024-02-24T05:04:29.952ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
10152024-02-24T05:04:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 9 reason = Timeout
10162024-02-24T05:04:29.952ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 9 reason = Timeout
10172024-02-24T05:04:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 9 reason = Timeout
10182024-02-24T05:04:29.952ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
10192024-02-24T05:04:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 11 reason = Timeout
10202024-02-24T05:04:29.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
10212024-02-24T05:04:29.952ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10222024-02-24T05:04:29.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
10232024-02-24T05:04:29.952ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
10242024-02-24T05:04:29.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
10252024-02-24T05:04:29.953ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 44 reason = Timeout
10262024-02-24T05:04:29.953ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 16 reason = Timeout
10272024-02-24T05:04:29.953ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
10282024-02-24T05:04:29.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
10292024-02-24T05:04:29.953ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 16 reason = Timeout
10302024-02-24T05:04:29.953ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
10312024-02-24T05:04:29.953ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 16 reason = Timeout
10322024-02-24T05:04:29.953ZWARNnexus (BackgroundTasks): phantom disk task started background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:46
10332024-02-24T05:04:29.953ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("49aa1114-08ad-4373-9921-df79e027bb19.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("680000fb-bd80-416d-9626-20dd6947323d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("680000fb-bd80-416d-9626-20dd6947323d.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:101::9)) }], valid_until: Instant { tv_sec: 13978, tv_nsec: 350844225 } })
10342024-02-24T05:04:29.953ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 16 reason = Timeout
10352024-02-24T05:04:29.953ZWARNnexus (BackgroundTasks): region replacement task started background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:44
10362024-02-24T05:04:29.953ZWARNnexus (BackgroundTasks): region replacement task done background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:48
10372024-02-24T05:04:29.953ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 9.76\u{b5}s iteration = 16
10382024-02-24T05:04:29.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
10392024-02-24T05:04:29.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
10402024-02-24T05:04:29.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
10412024-02-24T05:04:29.954ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10422024-02-24T05:04:29.954ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("5012e2a2-63d7-499d-af57-9f07ffa15664.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("c91d9fea-0a9d-4406-951b-fed674d2cdbd.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("e7877af7-833e-4a24-82d3-cf26f405dfbf.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("e7877af7-833e-4a24-82d3-cf26f405dfbf.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:3::1)) }], valid_until: Instant { tv_sec: 13978, tv_nsec: 351928009 } })
10432024-02-24T05:04:29.954ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10442024-02-24T05:04:29.955ZDEBGnexus (DpdClient): client request body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
10452024-02-24T05:04:29.955ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10462024-02-24T05:04:29.956ZDEBGnexus: client request MgAdminClient = [fd00:1122:3344:101::2]:4676 body = None method = GET uri = http://[fd00:1122:3344:101::2]:4676/bfd/peers
10472024-02-24T05:04:29.956ZWARNnexus (BackgroundTasks): phantom disk task done background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:92
10482024-02-24T05:04:29.956ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 2.930118ms iteration = 16
10492024-02-24T05:04:29.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
10502024-02-24T05:04:29.957ZDEBGnexus (DpdClient): client response result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "bfad477b-f87c-4ad2-b2dc-2b0b8dc7df68", "content-length": "1", "date": "Sat, 24 Feb 2024 05:04:29 GMT"} })
10512024-02-24T05:04:29.957ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
10522024-02-24T05:04:29.958ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
10532024-02-24T05:04:29.958ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
10542024-02-24T05:04:29.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
10552024-02-24T05:04:29.958ZDEBGnexus: client response MgAdminClient = [fd00:1122:3344:101::2]:4676 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(4676), path: "/bfd/peers", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "35e1449e-8f2e-4ca8-87be-7070e2219e0e", "content-length": "2", "date": "Sat, 24 Feb 2024 05:04:29 GMT"} })
10562024-02-24T05:04:29.958ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 4.704372ms iteration = 16
10572024-02-24T05:04:29.958ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
10582024-02-24T05:04:29.958ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
10592024-02-24T05:04:29.958ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
10602024-02-24T05:04:29.958ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
10612024-02-24T05:04:29.958ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } dns_group = external
10622024-02-24T05:04:29.958ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 6.04479ms iteration = 9
10632024-02-24T05:04:29.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
10642024-02-24T05:04:29.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
10652024-02-24T05:04:29.958ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } dns_group = internal
10662024-02-24T05:04:29.959ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 6.680188ms iteration = 9
10672024-02-24T05:04:29.959ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
10682024-02-24T05:04:29.959ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 6.135441ms iteration = 16
10692024-02-24T05:04:29.959ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
10702024-02-24T05:04:29.959ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10712024-02-24T05:04:29.960ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
10722024-02-24T05:04:29.960ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
10732024-02-24T05:04:29.960ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
10742024-02-24T05:04:29.960ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
10752024-02-24T05:04:29.960ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
10762024-02-24T05:04:29.960ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
10772024-02-24T05:04:29.960ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
10782024-02-24T05:04:29.960ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
10792024-02-24T05:04:29.960ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
10802024-02-24T05:04:29.961ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10812024-02-24T05:04:29.961ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 8.305449ms iteration = 44
10822024-02-24T05:04:29.961ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10832024-02-24T05:04:29.963ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
10842024-02-24T05:04:29.963ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
10852024-02-24T05:04:29.963ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
10862024-02-24T05:04:29.964ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
10872024-02-24T05:04:29.964ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
10882024-02-24T05:04:29.964ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
10892024-02-24T05:04:29.964ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = ed7b83d2-8dc6-441f-a2e2-0fbc5ae652e9 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
10902024-02-24T05:04:29.964ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
10912024-02-24T05:04:29.965ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
10922024-02-24T05:04:29.965ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
10932024-02-24T05:04:29.965ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
10942024-02-24T05:04:29.965ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
10952024-02-24T05:04:29.965ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
10962024-02-24T05:04:29.966ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10972024-02-24T05:04:29.966ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
10982024-02-24T05:04:29.966ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
10992024-02-24T05:04:29.967ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
11002024-02-24T05:04:29.967ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 6b78b467-60a6-4ddf-874b-97709602a8c7 dns_zone_name = oxide.test found_so_far = 0 version = 2
11012024-02-24T05:04:29.967ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
11022024-02-24T05:04:29.967ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
11032024-02-24T05:04:29.968ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11042024-02-24T05:04:29.969ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11052024-02-24T05:04:29.970ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
11062024-02-24T05:04:29.970ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
11072024-02-24T05:04:29.970ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
11082024-02-24T05:04:29.971ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
11092024-02-24T05:04:29.971ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
11102024-02-24T05:04:29.972ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
11112024-02-24T05:04:29.973ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = ed7b83d2-8dc6-441f-a2e2-0fbc5ae652e9 dns_zone_name = control-plane.oxide.internal found_so_far = 49 version = 1
11122024-02-24T05:04:29.973ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
11132024-02-24T05:04:29.973ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-02-24 04:57:46.561988 UTC dns_group = internal generation = 1
11142024-02-24T05:04:29.973ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 23.723853ms iteration = 9
11152024-02-24T05:04:29.973ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
11162024-02-24T05:04:29.973ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
11172024-02-24T05:04:29.973ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
11182024-02-24T05:04:29.974ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 6b78b467-60a6-4ddf-874b-97709602a8c7 dns_zone_name = oxide.test found_so_far = 1 version = 2
11192024-02-24T05:04:29.974ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
11202024-02-24T05:04:29.974ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-02-24 04:57:46.984314 UTC dns_group = external generation = 2
11212024-02-24T05:04:29.974ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 22.648559ms iteration = 9
11222024-02-24T05:04:29.974ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 16 reason = Timeout
11232024-02-24T05:04:29.974ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
11242024-02-24T05:04:29.975ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
11252024-02-24T05:04:29.976ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11262024-02-24T05:04:29.977ZINFOnexus (BackgroundTasks): found Silo/DNS/TLS config (changed) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-02-24T04:57:30.091889Z, time_modified: 2024-02-24T04:57:30.091889Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 14225503-e2f4-4614-85f2-4171f2c0cc28, db_silo: Silo { identity: SiloIdentity { id: 14225503-e2f4-4614-85f2-4171f2c0cc28, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-02-24T04:57:46.956873Z, time_modified: 2024-02-24T04:57:46.956873Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(4)) }, tls_certs: [TlsCertificate { digest: "1a902f55b5e588121c22f4a3e8f7791cdc445ad47d5a2a57211259fe1a5b280d" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 14225503-e2f4-4614-85f2-4171f2c0cc28, db_silo: Silo { identity: SiloIdentity { id: 14225503-e2f4-4614-85f2-4171f2c0cc28, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-02-24T04:57:46.956873Z, time_modified: 2024-02-24T04:57:46.956873Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(4)) }, tls_certs: [TlsCertificate { digest: "1a902f55b5e588121c22f4a3e8f7791cdc445ad47d5a2a57211259fe1a5b280d" }] }) } file = nexus/src/app/background/external_endpoints.rs:101
11272024-02-24T05:04:29.978ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 25.627848ms iteration = 9
11282024-02-24T05:04:29.978ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 9 reason = Timeout
11292024-02-24T05:04:29.978ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/blueprint_execution.rs:48 reason = no blueprint
11302024-02-24T05:04:29.978ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 9.56\u{b5}s iteration = 9
11312024-02-24T05:04:29.979ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
11322024-02-24T05:04:29.979ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
11332024-02-24T05:04:29.979ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
11342024-02-24T05:04:29.980ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
11352024-02-24T05:04:29.980ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
11362024-02-24T05:04:29.991ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
11372024-02-24T05:04:29.992ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
11382024-02-24T05:04:29.993ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
11392024-02-24T05:04:29.993ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
11402024-02-24T05:04:29.994ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11412024-02-24T05:04:29.996ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 318de5c0-d9bb-4803-90de-c5f48d23e50d, lookup_type: ById(318de5c0-d9bb-4803-90de-c5f48d23e50d) } result = Ok(())
11422024-02-24T05:04:29.996ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
11432024-02-24T05:04:29.996ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
11442024-02-24T05:04:29.998ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 23.49731ms iteration = 16
11452024-02-24T05:04:30.017ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
11462024-02-24T05:04:30.019ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
11472024-02-24T05:04:30.075ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
11482024-02-24T05:04:30.079ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
11492024-02-24T05:04:30.079ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::8)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "e1ba6493-1e99-46bc-947c-ea9e6f26a3ef", "date": "Sat, 24 Feb 2024 05:04:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
11502024-02-24T05:04:30.079ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
11512024-02-24T05:04:30.081ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::9)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "ee6c0e71-56a6-4269-9bd4-6a416a729414", "date": "Sat, 24 Feb 2024 05:04:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
11522024-02-24T05:04:30.081ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
11532024-02-24T05:04:30.082ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
11542024-02-24T05:04:30.082ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 130.125376ms iteration = 11
11552024-02-24T05:04:30.106ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
11562024-02-24T05:04:30.107ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:1::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "9af31cac-28eb-4437-a511-77bb6b413354", "date": "Sat, 24 Feb 2024 05:04:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
11572024-02-24T05:04:30.107ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
11582024-02-24T05:04:30.108ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:3::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "9de4d0e1-5f87-4a76-8116-7d0492b8e82c", "date": "Sat, 24 Feb 2024 05:04:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
11592024-02-24T05:04:30.108ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
11602024-02-24T05:04:30.109ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:2::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "b7030df0-2cb7-4d43-9f74-6c87395f71df", "date": "Sat, 24 Feb 2024 05:04:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
11612024-02-24T05:04:30.109ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
11622024-02-24T05:04:30.111ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
11632024-02-24T05:04:30.111ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 159.676935ms iteration = 11
11642024-02-24T05:04:30.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 177 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = d3861ab4-e274-4942-a868-00fd8721f7af response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
11652024-02-24T05:04:39.953ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 45 reason = Timeout
11662024-02-24T05:04:39.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
11672024-02-24T05:04:39.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
11682024-02-24T05:04:39.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11692024-02-24T05:04:39.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
11702024-02-24T05:04:39.957ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
11712024-02-24T05:04:39.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
11722024-02-24T05:04:39.958ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 5.553864ms iteration = 45
11732024-02-24T05:04:40.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 172 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 401a0611-2405-400b-83d2-095fb95eefe2 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
11742024-02-24T05:04:49.953ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 46 reason = Timeout
11752024-02-24T05:04:49.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
11762024-02-24T05:04:49.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
11772024-02-24T05:04:49.955ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11782024-02-24T05:04:49.958ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
11792024-02-24T05:04:49.958ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
11802024-02-24T05:04:49.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
11812024-02-24T05:04:49.959ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 5.909848ms iteration = 46
11822024-02-24T05:04:50.205ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 801 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 5257381e-7391-4a81-a53c-b5579cc9776e response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
11832024-02-24T05:04:59.953ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 47 reason = Timeout
11842024-02-24T05:04:59.953ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 17 reason = Timeout
11852024-02-24T05:04:59.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
11862024-02-24T05:04:59.953ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
11872024-02-24T05:04:59.953ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 17 reason = Timeout
11882024-02-24T05:04:59.953ZWARNnexus (BackgroundTasks): phantom disk task started background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:46
11892024-02-24T05:04:59.953ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 17 reason = Timeout
11902024-02-24T05:04:59.953ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
11912024-02-24T05:04:59.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
11922024-02-24T05:04:59.954ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 17 reason = Timeout
11932024-02-24T05:04:59.954ZWARNnexus (BackgroundTasks): region replacement task started background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:44
11942024-02-24T05:04:59.954ZWARNnexus (BackgroundTasks): region replacement task done background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:48
11952024-02-24T05:04:59.954ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 8.31\u{b5}s iteration = 17
11962024-02-24T05:04:59.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
11972024-02-24T05:04:59.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
11982024-02-24T05:04:59.955ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11992024-02-24T05:04:59.955ZDEBGnexus (DpdClient): client request body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
12002024-02-24T05:04:59.955ZDEBGnexus: client request MgAdminClient = [fd00:1122:3344:101::2]:4676 body = None method = GET uri = http://[fd00:1122:3344:101::2]:4676/bfd/peers
12012024-02-24T05:04:59.955ZDEBGnexus (DpdClient): client response result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "748188b7-182c-482d-86a5-4bb6cd414f20", "content-length": "2", "date": "Sat, 24 Feb 2024 05:04:59 GMT"} })
12022024-02-24T05:04:59.955ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
12032024-02-24T05:04:59.955ZDEBGnexus: client response MgAdminClient = [fd00:1122:3344:101::2]:4676 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(4676), path: "/bfd/peers", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "4cbb40eb-be88-46be-bcb6-e4e1726fd89a", "content-length": "2", "date": "Sat, 24 Feb 2024 05:04:59 GMT"} })
12042024-02-24T05:04:59.955ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 2.153858ms iteration = 17
12052024-02-24T05:04:59.956ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
12062024-02-24T05:04:59.957ZWARNnexus (BackgroundTasks): phantom disk task done background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:92
12072024-02-24T05:04:59.957ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.228353ms iteration = 17
12082024-02-24T05:04:59.957ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 3.72649ms iteration = 17
12092024-02-24T05:04:59.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
12102024-02-24T05:04:59.957ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
12112024-02-24T05:04:59.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
12122024-02-24T05:04:59.959ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 5.575993ms iteration = 47
12132024-02-24T05:04:59.975ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 17 reason = Timeout
12142024-02-24T05:04:59.975ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
12152024-02-24T05:04:59.975ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
12162024-02-24T05:04:59.976ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12172024-02-24T05:04:59.979ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
12182024-02-24T05:04:59.979ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
12192024-02-24T05:04:59.979ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
12202024-02-24T05:04:59.980ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
12212024-02-24T05:04:59.980ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
12222024-02-24T05:04:59.994ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
12232024-02-24T05:04:59.994ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
12242024-02-24T05:04:59.995ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
12252024-02-24T05:04:59.995ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
12262024-02-24T05:04:59.996ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12272024-02-24T05:04:59.999ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 318de5c0-d9bb-4803-90de-c5f48d23e50d, lookup_type: ById(318de5c0-d9bb-4803-90de-c5f48d23e50d) } result = Ok(())
12282024-02-24T05:04:59.999ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
12292024-02-24T05:04:59.999ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
12302024-02-24T05:05:00.000ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 24.738526ms iteration = 17
12312024-02-24T05:05:00.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 331 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 4feb3790-cc86-4bbf-94fa-4a51cb061f4c response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
12322024-02-24T05:05:09.953ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 48 reason = Timeout
12332024-02-24T05:05:09.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
12342024-02-24T05:05:09.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
12352024-02-24T05:05:09.955ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12362024-02-24T05:05:09.958ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
12372024-02-24T05:05:09.958ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
12382024-02-24T05:05:09.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
12392024-02-24T05:05:09.960ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 6.216462ms iteration = 48
12402024-02-24T05:05:10.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 244 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 5d0cc4e4-f493-4fb6-99e3-e85a050df71a response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
12412024-02-24T05:05:19.953ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 49 reason = Timeout
12422024-02-24T05:05:19.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
12432024-02-24T05:05:19.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
12442024-02-24T05:05:19.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12452024-02-24T05:05:19.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
12462024-02-24T05:05:19.957ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
12472024-02-24T05:05:19.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
12482024-02-24T05:05:19.964ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 11.139067ms iteration = 49
12492024-02-24T05:05:20.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 243 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = e304e48c-160f-4322-b12a-531933204964 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
12502024-02-24T05:05:29.949ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 10 reason = Timeout
12512024-02-24T05:05:29.949ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
12522024-02-24T05:05:29.949ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
12532024-02-24T05:05:29.950ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
12542024-02-24T05:05:29.950ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
12552024-02-24T05:05:29.950ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
12562024-02-24T05:05:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 12 reason = Timeout
12572024-02-24T05:05:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 10 reason = Timeout
12582024-02-24T05:05:29.952ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
12592024-02-24T05:05:29.952ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
12602024-02-24T05:05:29.952ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 10 reason = Timeout
12612024-02-24T05:05:29.952ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
12622024-02-24T05:05:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 12 reason = Timeout
12632024-02-24T05:05:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 10 reason = Timeout
12642024-02-24T05:05:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 10 reason = Timeout
12652024-02-24T05:05:29.952ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12662024-02-24T05:05:29.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
12672024-02-24T05:05:29.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
12682024-02-24T05:05:29.952ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
12692024-02-24T05:05:29.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
12702024-02-24T05:05:29.953ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 50 reason = Timeout
12712024-02-24T05:05:29.953ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 18 reason = Timeout
12722024-02-24T05:05:29.953ZWARNnexus (BackgroundTasks): phantom disk task started background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:46
12732024-02-24T05:05:29.953ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 18 reason = Timeout
12742024-02-24T05:05:29.953ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
12752024-02-24T05:05:29.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
12762024-02-24T05:05:29.953ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 18 reason = Timeout
12772024-02-24T05:05:29.953ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
12782024-02-24T05:05:29.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
12792024-02-24T05:05:29.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
12802024-02-24T05:05:29.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
12812024-02-24T05:05:29.953ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("5012e2a2-63d7-499d-af57-9f07ffa15664.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("c91d9fea-0a9d-4406-951b-fed674d2cdbd.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("e7877af7-833e-4a24-82d3-cf26f405dfbf.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("e7877af7-833e-4a24-82d3-cf26f405dfbf.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:3::1)) }], valid_until: Instant { tv_sec: 14038, tv_nsec: 350840336 } })
12822024-02-24T05:05:29.954ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("49aa1114-08ad-4373-9921-df79e027bb19.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("680000fb-bd80-416d-9626-20dd6947323d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("680000fb-bd80-416d-9626-20dd6947323d.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:101::9)) }], valid_until: Instant { tv_sec: 14038, tv_nsec: 350840376 } })
12832024-02-24T05:05:29.954ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 18 reason = Timeout
12842024-02-24T05:05:29.954ZWARNnexus (BackgroundTasks): region replacement task started background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:44
12852024-02-24T05:05:29.954ZWARNnexus (BackgroundTasks): region replacement task done background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:48
12862024-02-24T05:05:29.954ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 11.11\u{b5}s iteration = 18
12872024-02-24T05:05:29.954ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12882024-02-24T05:05:29.954ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12892024-02-24T05:05:29.955ZDEBGnexus (DpdClient): client request body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
12902024-02-24T05:05:29.955ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12912024-02-24T05:05:29.956ZDEBGnexus: client request MgAdminClient = [fd00:1122:3344:101::2]:4676 body = None method = GET uri = http://[fd00:1122:3344:101::2]:4676/bfd/peers
12922024-02-24T05:05:29.956ZDEBGnexus (DpdClient): client response result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "300198ef-718e-482f-8aa3-45deda2bd1ed", "content-length": "2", "date": "Sat, 24 Feb 2024 05:05:29 GMT"} })
12932024-02-24T05:05:29.956ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
12942024-02-24T05:05:29.956ZWARNnexus (BackgroundTasks): phantom disk task done background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:92
12952024-02-24T05:05:29.956ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.01588ms iteration = 18
12962024-02-24T05:05:29.956ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
12972024-02-24T05:05:29.957ZDEBGnexus: client response MgAdminClient = [fd00:1122:3344:101::2]:4676 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(4676), path: "/bfd/peers", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "181cabc1-15c9-462d-9cca-88cc5bf1ad0d", "content-length": "2", "date": "Sat, 24 Feb 2024 05:05:29 GMT"} })
12982024-02-24T05:05:29.957ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 3.75196ms iteration = 18
12992024-02-24T05:05:29.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
13002024-02-24T05:05:29.957ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
13012024-02-24T05:05:29.957ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
13022024-02-24T05:05:29.957ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } dns_group = external
13032024-02-24T05:05:29.957ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 5.195338ms iteration = 10
13042024-02-24T05:05:29.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
13052024-02-24T05:05:29.958ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 4.663122ms iteration = 18
13062024-02-24T05:05:29.958ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
13072024-02-24T05:05:29.958ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
13082024-02-24T05:05:29.958ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
13092024-02-24T05:05:29.958ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
13102024-02-24T05:05:29.958ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } dns_group = internal
13112024-02-24T05:05:29.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
13122024-02-24T05:05:29.958ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 6.367294ms iteration = 10
13132024-02-24T05:05:29.959ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
13142024-02-24T05:05:29.959ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
13152024-02-24T05:05:29.959ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
13162024-02-24T05:05:29.959ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
13172024-02-24T05:05:29.960ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
13182024-02-24T05:05:29.960ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
13192024-02-24T05:05:29.960ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
13202024-02-24T05:05:29.960ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
13212024-02-24T05:05:29.960ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
13222024-02-24T05:05:29.960ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
13232024-02-24T05:05:29.960ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
13242024-02-24T05:05:29.960ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
13252024-02-24T05:05:29.961ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
13262024-02-24T05:05:29.961ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
13272024-02-24T05:05:29.962ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
13282024-02-24T05:05:29.962ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
13292024-02-24T05:05:29.963ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
13302024-02-24T05:05:29.965ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
13312024-02-24T05:05:29.965ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = ed7b83d2-8dc6-441f-a2e2-0fbc5ae652e9 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
13322024-02-24T05:05:29.965ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
13332024-02-24T05:05:29.965ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
13342024-02-24T05:05:29.965ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
13352024-02-24T05:05:29.965ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
13362024-02-24T05:05:29.965ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
13372024-02-24T05:05:29.965ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
13382024-02-24T05:05:29.965ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
13392024-02-24T05:05:29.965ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
13402024-02-24T05:05:29.966ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 12.944891ms iteration = 50
13412024-02-24T05:05:29.966ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
13422024-02-24T05:05:29.967ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
13432024-02-24T05:05:29.967ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
13442024-02-24T05:05:29.967ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 6b78b467-60a6-4ddf-874b-97709602a8c7 dns_zone_name = oxide.test found_so_far = 0 version = 2
13452024-02-24T05:05:29.967ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
13462024-02-24T05:05:29.967ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
13472024-02-24T05:05:29.967ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
13482024-02-24T05:05:29.968ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
13492024-02-24T05:05:29.969ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
13502024-02-24T05:05:29.971ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
13512024-02-24T05:05:29.972ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
13522024-02-24T05:05:29.972ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
13532024-02-24T05:05:29.972ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
13542024-02-24T05:05:29.972ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
13552024-02-24T05:05:29.973ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
13562024-02-24T05:05:29.974ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = ed7b83d2-8dc6-441f-a2e2-0fbc5ae652e9 dns_zone_name = control-plane.oxide.internal found_so_far = 49 version = 1
13572024-02-24T05:05:29.975ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
13582024-02-24T05:05:29.975ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-02-24 04:57:46.561988 UTC dns_group = internal generation = 1
13592024-02-24T05:05:29.975ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 25.150891ms iteration = 10
13602024-02-24T05:05:29.975ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
13612024-02-24T05:05:29.975ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
13622024-02-24T05:05:29.975ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 18 reason = Timeout
13632024-02-24T05:05:29.975ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
13642024-02-24T05:05:29.975ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
13652024-02-24T05:05:29.975ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
13662024-02-24T05:05:29.976ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
13672024-02-24T05:05:29.976ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 6b78b467-60a6-4ddf-874b-97709602a8c7 dns_zone_name = oxide.test found_so_far = 1 version = 2
13682024-02-24T05:05:29.976ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
13692024-02-24T05:05:29.976ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-02-24 04:57:46.984314 UTC dns_group = external generation = 2
13702024-02-24T05:05:29.976ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 24.630175ms iteration = 10
13712024-02-24T05:05:29.978ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-02-24T04:57:30.091889Z, time_modified: 2024-02-24T04:57:30.091889Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 14225503-e2f4-4614-85f2-4171f2c0cc28, db_silo: Silo { identity: SiloIdentity { id: 14225503-e2f4-4614-85f2-4171f2c0cc28, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-02-24T04:57:46.956873Z, time_modified: 2024-02-24T04:57:46.956873Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(4)) }, tls_certs: [TlsCertificate { digest: "1a902f55b5e588121c22f4a3e8f7791cdc445ad47d5a2a57211259fe1a5b280d" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 14225503-e2f4-4614-85f2-4171f2c0cc28, db_silo: Silo { identity: SiloIdentity { id: 14225503-e2f4-4614-85f2-4171f2c0cc28, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-02-24T04:57:46.956873Z, time_modified: 2024-02-24T04:57:46.956873Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(4)) }, tls_certs: [TlsCertificate { digest: "1a902f55b5e588121c22f4a3e8f7791cdc445ad47d5a2a57211259fe1a5b280d" }] }) }
13722024-02-24T05:05:29.978ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 26.324098ms iteration = 10
13732024-02-24T05:05:29.978ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 10 reason = Timeout
13742024-02-24T05:05:29.978ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/blueprint_execution.rs:48 reason = no blueprint
13752024-02-24T05:05:29.978ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 11.97\u{b5}s iteration = 10
13762024-02-24T05:05:29.979ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
13772024-02-24T05:05:29.979ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
13782024-02-24T05:05:29.979ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
13792024-02-24T05:05:29.981ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
13802024-02-24T05:05:29.981ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
13812024-02-24T05:05:29.997ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
13822024-02-24T05:05:29.997ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
13832024-02-24T05:05:29.998ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
13842024-02-24T05:05:29.998ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
13852024-02-24T05:05:29.999ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
13862024-02-24T05:05:30.002ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 318de5c0-d9bb-4803-90de-c5f48d23e50d, lookup_type: ById(318de5c0-d9bb-4803-90de-c5f48d23e50d) } result = Ok(())
13872024-02-24T05:05:30.002ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
13882024-02-24T05:05:30.002ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
13892024-02-24T05:05:30.004ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 28.648968ms iteration = 18
13902024-02-24T05:05:30.020ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
13912024-02-24T05:05:30.021ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
13922024-02-24T05:05:30.085ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
13932024-02-24T05:05:30.085ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::8)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "634aa034-cd88-4f5e-804a-f3315bedeca5", "date": "Sat, 24 Feb 2024 05:05:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
13942024-02-24T05:05:30.085ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
13952024-02-24T05:05:30.087ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::9)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "36aaed3f-d8cd-4619-810e-a238c2e58e32", "date": "Sat, 24 Feb 2024 05:05:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
13962024-02-24T05:05:30.087ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
13972024-02-24T05:05:30.087ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
13982024-02-24T05:05:30.088ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
13992024-02-24T05:05:30.088ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 136.012483ms iteration = 12
14002024-02-24T05:05:30.117ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
14012024-02-24T05:05:30.117ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:1::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "c85fc300-3e79-4f5c-b502-c7ca5f7aff17", "date": "Sat, 24 Feb 2024 05:05:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
14022024-02-24T05:05:30.117ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
14032024-02-24T05:05:30.119ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:2::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "141c8511-99d1-4073-a408-12da4df0c194", "date": "Sat, 24 Feb 2024 05:05:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
14042024-02-24T05:05:30.119ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
14052024-02-24T05:05:30.121ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:3::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "318acea8-b650-4d2d-8068-9d42b3c558e8", "date": "Sat, 24 Feb 2024 05:05:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
14062024-02-24T05:05:30.121ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
14072024-02-24T05:05:30.123ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
14082024-02-24T05:05:30.123ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 171.202997ms iteration = 12
14092024-02-24T05:05:30.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 160 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 03a554f7-4507-4eae-accd-a8b84cbd042e response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
14102024-02-24T05:05:39.953ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 51 reason = Timeout
14112024-02-24T05:05:39.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
14122024-02-24T05:05:39.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
14132024-02-24T05:05:39.955ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
14142024-02-24T05:05:39.958ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
14152024-02-24T05:05:39.958ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
14162024-02-24T05:05:39.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
14172024-02-24T05:05:39.959ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 6.175602ms iteration = 51
14182024-02-24T05:05:40.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 262 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = c977a080-daaf-4565-a7aa-c3d82fc68e89 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
14192024-02-24T05:05:49.953ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 52 reason = Timeout
14202024-02-24T05:05:49.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
14212024-02-24T05:05:49.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
14222024-02-24T05:05:49.955ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
14232024-02-24T05:05:49.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
14242024-02-24T05:05:49.958ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
14252024-02-24T05:05:49.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
14262024-02-24T05:05:49.959ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 6.481386ms iteration = 52
14272024-02-24T05:05:50.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 144 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = b9c6f433-0322-4057-b0f1-b7599e7c69b8 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
14282024-02-24T05:05:59.953ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 53 reason = Timeout
14292024-02-24T05:05:59.953ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 19 reason = Timeout
14302024-02-24T05:05:59.953ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
14312024-02-24T05:05:59.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
14322024-02-24T05:05:59.953ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 19 reason = Timeout
14332024-02-24T05:05:59.953ZWARNnexus (BackgroundTasks): phantom disk task started background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:46
14342024-02-24T05:05:59.953ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 19 reason = Timeout
14352024-02-24T05:05:59.953ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
14362024-02-24T05:05:59.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
14372024-02-24T05:05:59.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
14382024-02-24T05:05:59.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
14392024-02-24T05:05:59.954ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 19 reason = Timeout
14402024-02-24T05:05:59.954ZWARNnexus (BackgroundTasks): region replacement task started background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:44
14412024-02-24T05:05:59.955ZWARNnexus (BackgroundTasks): region replacement task done background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:48
14422024-02-24T05:05:59.955ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 20.33\u{b5}s iteration = 19
14432024-02-24T05:05:59.955ZDEBGnexus (DpdClient): client request body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
14442024-02-24T05:05:59.955ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
14452024-02-24T05:05:59.955ZDEBGnexus (DpdClient): client response result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "daf27209-0d73-4d34-b9cc-fcd122f4bf63", "content-length": "2", "date": "Sat, 24 Feb 2024 05:05:59 GMT"} })
14462024-02-24T05:05:59.955ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
14472024-02-24T05:05:59.955ZDEBGnexus: client request MgAdminClient = [fd00:1122:3344:101::2]:4676 body = None method = GET uri = http://[fd00:1122:3344:101::2]:4676/bfd/peers
14482024-02-24T05:05:59.955ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
14492024-02-24T05:05:59.956ZDEBGnexus: client response MgAdminClient = [fd00:1122:3344:101::2]:4676 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(4676), path: "/bfd/peers", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "16878dce-69a6-4e1f-9d4d-d637e2edae27", "content-length": "2", "date": "Sat, 24 Feb 2024 05:05:59 GMT"} })
14502024-02-24T05:05:59.956ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 2.520783ms iteration = 19
14512024-02-24T05:05:59.956ZWARNnexus (BackgroundTasks): phantom disk task done background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:92
14522024-02-24T05:05:59.956ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 2.779937ms iteration = 19
14532024-02-24T05:05:59.957ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 3.553046ms iteration = 19
14542024-02-24T05:05:59.958ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
14552024-02-24T05:05:59.958ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
14562024-02-24T05:05:59.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
14572024-02-24T05:05:59.960ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 6.648487ms iteration = 53
14582024-02-24T05:05:59.976ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 19 reason = Timeout
14592024-02-24T05:05:59.976ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
14602024-02-24T05:05:59.976ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
14612024-02-24T05:05:59.977ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
14622024-02-24T05:05:59.980ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
14632024-02-24T05:05:59.980ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
14642024-02-24T05:05:59.980ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
14652024-02-24T05:05:59.981ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
14662024-02-24T05:05:59.981ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
14672024-02-24T05:05:59.997ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
14682024-02-24T05:05:59.998ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
14692024-02-24T05:05:59.999ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
14702024-02-24T05:05:59.999ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
14712024-02-24T05:06:00.000ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
14722024-02-24T05:06:00.003ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 318de5c0-d9bb-4803-90de-c5f48d23e50d, lookup_type: ById(318de5c0-d9bb-4803-90de-c5f48d23e50d) } result = Ok(())
14732024-02-24T05:06:00.003ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
14742024-02-24T05:06:00.003ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
14752024-02-24T05:06:00.004ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 28.653448ms iteration = 19
14762024-02-24T05:06:00.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 301 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 65cb9935-fe50-441b-a74e-e938812c9b26 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
14772024-02-24T05:06:09.953ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 54 reason = Timeout
14782024-02-24T05:06:09.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
14792024-02-24T05:06:09.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
14802024-02-24T05:06:09.955ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
14812024-02-24T05:06:09.958ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
14822024-02-24T05:06:09.958ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
14832024-02-24T05:06:09.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
14842024-02-24T05:06:09.959ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 6.014069ms iteration = 54
14852024-02-24T05:06:10.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 248 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = e070bdf6-b6ea-4ff4-96d3-a83a2349db90 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
14862024-02-24T05:06:19.954ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 55 reason = Timeout
14872024-02-24T05:06:19.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
14882024-02-24T05:06:19.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
14892024-02-24T05:06:19.955ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
14902024-02-24T05:06:19.958ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
14912024-02-24T05:06:19.958ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
14922024-02-24T05:06:19.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
14932024-02-24T05:06:19.960ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 6.014499ms iteration = 55
14942024-02-24T05:06:20.205ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 261 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = b735e9a2-8e94-4226-bb32-c1c17f90aa17 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
14952024-02-24T05:06:29.949ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 11 reason = Timeout
14962024-02-24T05:06:29.949ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
14972024-02-24T05:06:29.949ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
14982024-02-24T05:06:29.950ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
14992024-02-24T05:06:29.950ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
15002024-02-24T05:06:29.950ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
15012024-02-24T05:06:29.952ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15022024-02-24T05:06:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 13 reason = Timeout
15032024-02-24T05:06:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 11 reason = Timeout
15042024-02-24T05:06:29.952ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 11 reason = Timeout
15052024-02-24T05:06:29.952ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
15062024-02-24T05:06:29.952ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
15072024-02-24T05:06:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 13 reason = Timeout
15082024-02-24T05:06:29.952ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
15092024-02-24T05:06:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 11 reason = Timeout
15102024-02-24T05:06:29.952ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 11 reason = Timeout
15112024-02-24T05:06:29.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
15122024-02-24T05:06:29.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
15132024-02-24T05:06:29.953ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
15142024-02-24T05:06:29.953ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 56 reason = Timeout
15152024-02-24T05:06:29.953ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 20 reason = Timeout
15162024-02-24T05:06:29.953ZWARNnexus (BackgroundTasks): phantom disk task started background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:46
15172024-02-24T05:06:29.953ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 20 reason = Timeout
15182024-02-24T05:06:29.953ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 20 reason = Timeout
15192024-02-24T05:06:29.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
15202024-02-24T05:06:29.953ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
15212024-02-24T05:06:29.953ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
15222024-02-24T05:06:29.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
15232024-02-24T05:06:29.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
15242024-02-24T05:06:29.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
15252024-02-24T05:06:29.954ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 20 reason = Timeout
15262024-02-24T05:06:29.954ZWARNnexus (BackgroundTasks): region replacement task started background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:44
15272024-02-24T05:06:29.954ZWARNnexus (BackgroundTasks): region replacement task done background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:48
15282024-02-24T05:06:29.954ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 8.39\u{b5}s iteration = 20
15292024-02-24T05:06:29.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
15302024-02-24T05:06:29.954ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("5012e2a2-63d7-499d-af57-9f07ffa15664.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("c91d9fea-0a9d-4406-951b-fed674d2cdbd.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("e7877af7-833e-4a24-82d3-cf26f405dfbf.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("e7877af7-833e-4a24-82d3-cf26f405dfbf.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:3::1)) }], valid_until: Instant { tv_sec: 14098, tv_nsec: 351054241 } })
15312024-02-24T05:06:29.954ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("49aa1114-08ad-4373-9921-df79e027bb19.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("680000fb-bd80-416d-9626-20dd6947323d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("680000fb-bd80-416d-9626-20dd6947323d.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:101::9)) }], valid_until: Instant { tv_sec: 14098, tv_nsec: 351106792 } })
15322024-02-24T05:06:29.954ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15332024-02-24T05:06:29.955ZDEBGnexus (DpdClient): client request body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
15342024-02-24T05:06:29.955ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15352024-02-24T05:06:29.956ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15362024-02-24T05:06:29.956ZWARNnexus (BackgroundTasks): phantom disk task done background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:92
15372024-02-24T05:06:29.956ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 2.834367ms iteration = 20
15382024-02-24T05:06:29.956ZDEBGnexus: client request MgAdminClient = [fd00:1122:3344:101::2]:4676 body = None method = GET uri = http://[fd00:1122:3344:101::2]:4676/bfd/peers
15392024-02-24T05:06:29.956ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
15402024-02-24T05:06:29.957ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
15412024-02-24T05:06:29.957ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
15422024-02-24T05:06:29.957ZDEBGnexus (DpdClient): client response result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "6cd9278b-e9e1-4b1d-b74e-2f483abad45b", "content-length": "2", "date": "Sat, 24 Feb 2024 05:06:29 GMT"} })
15432024-02-24T05:06:29.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
15442024-02-24T05:06:29.957ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
15452024-02-24T05:06:29.958ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
15462024-02-24T05:06:29.958ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
15472024-02-24T05:06:29.958ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
15482024-02-24T05:06:29.958ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
15492024-02-24T05:06:29.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
15502024-02-24T05:06:29.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
15512024-02-24T05:06:29.958ZDEBGnexus: client response MgAdminClient = [fd00:1122:3344:101::2]:4676 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(4676), path: "/bfd/peers", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "1904d148-1469-4486-abeb-5248ea9b51b1", "content-length": "2", "date": "Sat, 24 Feb 2024 05:06:29 GMT"} })
15522024-02-24T05:06:29.958ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 5.079167ms iteration = 20
15532024-02-24T05:06:29.959ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
15542024-02-24T05:06:29.959ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
15552024-02-24T05:06:29.959ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15562024-02-24T05:06:29.959ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 5.958929ms iteration = 20
15572024-02-24T05:06:29.959ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } dns_group = external
15582024-02-24T05:06:29.959ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 7.302506ms iteration = 11
15592024-02-24T05:06:29.960ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15602024-02-24T05:06:29.960ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } dns_group = internal
15612024-02-24T05:06:29.960ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 8.386401ms iteration = 11
15622024-02-24T05:06:29.962ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
15632024-02-24T05:06:29.962ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
15642024-02-24T05:06:29.962ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
15652024-02-24T05:06:29.962ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
15662024-02-24T05:06:29.963ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
15672024-02-24T05:06:29.963ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
15682024-02-24T05:06:29.963ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
15692024-02-24T05:06:29.963ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
15702024-02-24T05:06:29.963ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
15712024-02-24T05:06:29.963ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
15722024-02-24T05:06:29.963ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
15732024-02-24T05:06:29.964ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 10.352236ms iteration = 56
15742024-02-24T05:06:29.964ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
15752024-02-24T05:06:29.964ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15762024-02-24T05:06:29.964ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
15772024-02-24T05:06:29.964ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
15782024-02-24T05:06:29.965ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
15792024-02-24T05:06:29.965ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
15802024-02-24T05:06:29.965ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = ed7b83d2-8dc6-441f-a2e2-0fbc5ae652e9 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
15812024-02-24T05:06:29.965ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
15822024-02-24T05:06:29.965ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
15832024-02-24T05:06:29.966ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
15842024-02-24T05:06:29.966ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
15852024-02-24T05:06:29.967ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15862024-02-24T05:06:29.967ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15872024-02-24T05:06:29.969ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
15882024-02-24T05:06:29.969ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
15892024-02-24T05:06:29.969ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
15902024-02-24T05:06:29.970ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
15912024-02-24T05:06:29.970ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
15922024-02-24T05:06:29.970ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
15932024-02-24T05:06:29.971ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
15942024-02-24T05:06:29.971ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
15952024-02-24T05:06:29.971ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
15962024-02-24T05:06:29.971ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 6b78b467-60a6-4ddf-874b-97709602a8c7 dns_zone_name = oxide.test found_so_far = 0 version = 2
15972024-02-24T05:06:29.971ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
15982024-02-24T05:06:29.971ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
15992024-02-24T05:06:29.971ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
16002024-02-24T05:06:29.972ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = ed7b83d2-8dc6-441f-a2e2-0fbc5ae652e9 dns_zone_name = control-plane.oxide.internal found_so_far = 49 version = 1
16012024-02-24T05:06:29.972ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
16022024-02-24T05:06:29.972ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-02-24 04:57:46.561988 UTC dns_group = internal generation = 1
16032024-02-24T05:06:29.972ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 23.232087ms iteration = 11
16042024-02-24T05:06:29.972ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16052024-02-24T05:06:29.975ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
16062024-02-24T05:06:29.975ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
16072024-02-24T05:06:29.975ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
16082024-02-24T05:06:29.976ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 20 reason = Timeout
16092024-02-24T05:06:29.976ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
16102024-02-24T05:06:29.976ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
16112024-02-24T05:06:29.976ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 6b78b467-60a6-4ddf-874b-97709602a8c7 dns_zone_name = oxide.test found_so_far = 1 version = 2
16122024-02-24T05:06:29.976ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
16132024-02-24T05:06:29.976ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-02-24 04:57:46.984314 UTC dns_group = external generation = 2
16142024-02-24T05:06:29.976ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 24.152949ms iteration = 11
16152024-02-24T05:06:29.977ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16162024-02-24T05:06:29.978ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 11 reason = Timeout
16172024-02-24T05:06:29.978ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/blueprint_execution.rs:48 reason = no blueprint
16182024-02-24T05:06:29.978ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 12.66\u{b5}s iteration = 11
16192024-02-24T05:06:29.980ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
16202024-02-24T05:06:29.980ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
16212024-02-24T05:06:29.980ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
16222024-02-24T05:06:29.981ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-02-24T04:57:30.091889Z, time_modified: 2024-02-24T04:57:30.091889Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 14225503-e2f4-4614-85f2-4171f2c0cc28, db_silo: Silo { identity: SiloIdentity { id: 14225503-e2f4-4614-85f2-4171f2c0cc28, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-02-24T04:57:46.956873Z, time_modified: 2024-02-24T04:57:46.956873Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(4)) }, tls_certs: [TlsCertificate { digest: "1a902f55b5e588121c22f4a3e8f7791cdc445ad47d5a2a57211259fe1a5b280d" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 14225503-e2f4-4614-85f2-4171f2c0cc28, db_silo: Silo { identity: SiloIdentity { id: 14225503-e2f4-4614-85f2-4171f2c0cc28, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-02-24T04:57:46.956873Z, time_modified: 2024-02-24T04:57:46.956873Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(4)) }, tls_certs: [TlsCertificate { digest: "1a902f55b5e588121c22f4a3e8f7791cdc445ad47d5a2a57211259fe1a5b280d" }] }) }
16232024-02-24T05:06:29.981ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 29.422598ms iteration = 11
16242024-02-24T05:06:29.981ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
16252024-02-24T05:06:29.982ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
16262024-02-24T05:06:29.993ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
16272024-02-24T05:06:29.993ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
16282024-02-24T05:06:29.994ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
16292024-02-24T05:06:29.995ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
16302024-02-24T05:06:29.996ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16312024-02-24T05:06:29.998ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 318de5c0-d9bb-4803-90de-c5f48d23e50d, lookup_type: ById(318de5c0-d9bb-4803-90de-c5f48d23e50d) } result = Ok(())
16322024-02-24T05:06:29.998ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
16332024-02-24T05:06:29.999ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
16342024-02-24T05:06:30.000ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 24.095158ms iteration = 20
16352024-02-24T05:06:30.022ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
16362024-02-24T05:06:30.028ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
16372024-02-24T05:06:30.084ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
16382024-02-24T05:06:30.084ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::9)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "959b60b5-1b11-4f27-8ee9-b3c3fb69fcdd", "date": "Sat, 24 Feb 2024 05:06:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
16392024-02-24T05:06:30.084ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
16402024-02-24T05:06:30.085ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::8)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "615b5851-7de9-4044-93dd-5c4af9113c2c", "date": "Sat, 24 Feb 2024 05:06:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
16412024-02-24T05:06:30.085ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
16422024-02-24T05:06:30.086ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::8]:5353, [fd00:1122:3344:101::9]:5353] }
16432024-02-24T05:06:30.086ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 134.039977ms iteration = 13
16442024-02-24T05:06:30.087ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
16452024-02-24T05:06:30.116ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
16462024-02-24T05:06:30.116ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:1::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "fda4caa1-e980-4a92-8005-cd74b6b4e162", "date": "Sat, 24 Feb 2024 05:06:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
16472024-02-24T05:06:30.116ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
16482024-02-24T05:06:30.117ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:2::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "e3138c37-ee65-4f18-a000-bf345d579256", "date": "Sat, 24 Feb 2024 05:06:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
16492024-02-24T05:06:30.117ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
16502024-02-24T05:06:30.119ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:3::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "1fe4c8a3-dd66-496a-97e3-a16e1a9a5233", "date": "Sat, 24 Feb 2024 05:06:30 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
16512024-02-24T05:06:30.119ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
16522024-02-24T05:06:30.121ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
16532024-02-24T05:06:30.121ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 168.693994ms iteration = 13
16542024-02-24T05:06:30.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 184 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 42def15c-05fb-4edc-b9ae-2de345f37a94 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
16552024-02-24T05:06:39.954ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 57 reason = Timeout
16562024-02-24T05:06:39.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
16572024-02-24T05:06:39.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
16582024-02-24T05:06:39.955ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16592024-02-24T05:06:39.958ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
16602024-02-24T05:06:39.958ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
16612024-02-24T05:06:39.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
16622024-02-24T05:06:39.959ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 5.885838ms iteration = 57
16632024-02-24T05:06:40.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 200 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 00539fee-a610-4a47-a268-95716564c041 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
16642024-02-24T05:06:49.954ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 58 reason = Timeout
16652024-02-24T05:06:49.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
16662024-02-24T05:06:49.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
16672024-02-24T05:06:49.956ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16682024-02-24T05:06:49.958ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
16692024-02-24T05:06:49.958ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
16702024-02-24T05:06:49.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
16712024-02-24T05:06:49.959ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 5.864787ms iteration = 58
16722024-02-24T05:06:50.205ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 246 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 365a17ed-a80f-4e4a-8382-5ad567f1da30 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
16732024-02-24T05:06:59.953ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 59 reason = Timeout
16742024-02-24T05:06:59.953ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
16752024-02-24T05:06:59.953ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 21 reason = Timeout
16762024-02-24T05:06:59.953ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 21 reason = Timeout
16772024-02-24T05:06:59.953ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
16782024-02-24T05:06:59.953ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
16792024-02-24T05:06:59.953ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 21 reason = Timeout
16802024-02-24T05:06:59.953ZWARNnexus (BackgroundTasks): phantom disk task started background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:46
16812024-02-24T05:06:59.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
16822024-02-24T05:06:59.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
16832024-02-24T05:06:59.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
16842024-02-24T05:06:59.954ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 21 reason = Timeout
16852024-02-24T05:06:59.954ZWARNnexus (BackgroundTasks): region replacement task started background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:44
16862024-02-24T05:06:59.954ZWARNnexus (BackgroundTasks): region replacement task done background_task = region_replacement file = nexus/src/app/background/region_replacement.rs:48
16872024-02-24T05:06:59.954ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 10.84\u{b5}s iteration = 21
16882024-02-24T05:06:59.955ZDEBGnexus (DpdClient): client request body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
16892024-02-24T05:06:59.955ZDEBGnexus (DpdClient): client response result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "aa48f774-f769-4827-8829-9bf828dffd68", "content-length": "2", "date": "Sat, 24 Feb 2024 05:06:59 GMT"} })
16902024-02-24T05:06:59.955ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
16912024-02-24T05:06:59.955ZDEBGnexus: client request MgAdminClient = [fd00:1122:3344:101::2]:4676 body = None method = GET uri = http://[fd00:1122:3344:101::2]:4676/bfd/peers
16922024-02-24T05:06:59.955ZDEBGnexus: client response MgAdminClient = [fd00:1122:3344:101::2]:4676 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(4676), path: "/bfd/peers", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "bdb9dc66-b7a1-444a-8546-8c16efd36dbe", "content-length": "2", "date": "Sat, 24 Feb 2024 05:06:59 GMT"} })
16932024-02-24T05:06:59.955ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 2.247429ms iteration = 21
16942024-02-24T05:06:59.956ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
16952024-02-24T05:06:59.956ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16962024-02-24T05:06:59.956ZWARNnexus (BackgroundTasks): phantom disk task done background_task = phantom_disks file = nexus/src/app/background/phantom_disks.rs:92
16972024-02-24T05:06:59.956ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 2.990359ms iteration = 21
16982024-02-24T05:06:59.957ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 3.902741ms iteration = 21
16992024-02-24T05:06:59.958ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
17002024-02-24T05:06:59.958ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
17012024-02-24T05:06:59.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
17022024-02-24T05:06:59.960ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 6.447765ms iteration = 59
17032024-02-24T05:06:59.975ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 21 reason = Timeout
17042024-02-24T05:06:59.975ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
17052024-02-24T05:06:59.976ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
17062024-02-24T05:06:59.977ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
17072024-02-24T05:06:59.979ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
17082024-02-24T05:06:59.979ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
17092024-02-24T05:06:59.980ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
17102024-02-24T05:06:59.981ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
17112024-02-24T05:06:59.981ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
17122024-02-24T05:06:59.992ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
17132024-02-24T05:06:59.992ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
17142024-02-24T05:06:59.993ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
17152024-02-24T05:06:59.994ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
17162024-02-24T05:06:59.995ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
17172024-02-24T05:06:59.997ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 318de5c0-d9bb-4803-90de-c5f48d23e50d, lookup_type: ById(318de5c0-d9bb-4803-90de-c5f48d23e50d) } result = Ok(())
17182024-02-24T05:06:59.997ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
17192024-02-24T05:06:59.997ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
17202024-02-24T05:06:59.998ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 23.046364ms iteration = 21
17212024-02-24T05:07:00.204ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 236 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = c2340d9f-e7e4-4a1a-9150-152f9ed89ac2 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
17222024-02-24T05:07:09.954ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 60 reason = Timeout
17232024-02-24T05:07:09.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
17242024-02-24T05:07:09.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
17252024-02-24T05:07:09.955ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
17262024-02-24T05:07:09.958ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
17272024-02-24T05:07:09.958ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
17282024-02-24T05:07:09.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
17292024-02-24T05:07:09.960ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 6.019959ms iteration = 60
17302024-02-24T05:07:10.205ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 152 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 9deb1426-e5c2-40c3-af58-03a85ea24e29 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771
17312024-02-24T05:07:19.954ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 61 reason = Timeout
17322024-02-24T05:07:19.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
17332024-02-24T05:07:19.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
17342024-02-24T05:07:19.956ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
17352024-02-24T05:07:19.958ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
17362024-02-24T05:07:19.958ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
17372024-02-24T05:07:19.959ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
17382024-02-24T05:07:19.960ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 6.048899ms iteration = 61
17392024-02-24T05:07:20.205ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849 latency_us = 157 local_addr = [fd00:1122:3344:101::b]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:53877 req_id = 8eb93146-fa9b-4092-9844-8c3b07591dc7 response_code = 200 uri = /metrics/collect/fcbb0e99-4a51-4f9a-bb87-154aabcb9771