Max level shown:
12024-08-26T20:30:00.874ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 1129 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = c3d46a6c-0394-45a3-a8e4-85548a554387 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
22024-08-26T20:30:10.247ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 12 reason = Timeout
32024-08-26T20:30:10.247ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
42024-08-26T20:30:10.247ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
52024-08-26T20:30:10.247ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 12 reason = Timeout
62024-08-26T20:30:10.247ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
72024-08-26T20:30:10.247ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
82024-08-26T20:30:10.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
92024-08-26T20:30:10.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
102024-08-26T20:30:10.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
112024-08-26T20:30:10.250ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 3.11902ms iteration = 12 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
122024-08-26T20:30:10.251ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
132024-08-26T20:30:10.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
142024-08-26T20:30:10.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
152024-08-26T20:30:10.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
162024-08-26T20:30:10.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
172024-08-26T20:30:10.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
182024-08-26T20:30:10.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
192024-08-26T20:30:10.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
202024-08-26T20:30:10.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
212024-08-26T20:30:10.263ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
222024-08-26T20:30:10.263ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 16.426414ms iteration = 12 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
232024-08-26T20:30:10.873ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 208 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = bc0b89b3-966a-402a-9e7b-3855a9d6bda3 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
242024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 5 reason = Timeout
252024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 3 reason = Timeout
262024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 6 reason = Timeout
272024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
282024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 13 reason = Timeout
292024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 3 reason = Timeout
302024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
312024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 5 reason = Timeout
322024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
332024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 3 reason = Timeout
342024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 5 reason = Timeout
352024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 5 reason = Timeout
362024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 5 reason = Timeout
372024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
382024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 5 reason = Timeout
392024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
402024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
412024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 3 reason = Timeout
422024-08-26T20:30:20.247ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
432024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
442024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 5 reason = Timeout
452024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
462024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 3 reason = Timeout
472024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 4 reason = Timeout
482024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
492024-08-26T20:30:20.247ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:80 reason = blueprint disabled target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460
502024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
512024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 18.76\u{b5}s iteration = 4 status = {"error":"blueprint disabled","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460"}
522024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 3 reason = Timeout
532024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 3 reason = Timeout
542024-08-26T20:30:20.247ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-08-26 20:20:20.247547743 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
552024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
562024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 5 reason = Timeout
572024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 5 reason = Timeout
582024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
592024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
602024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
612024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
622024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 5 reason = Timeout
632024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
642024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 13 reason = Timeout
652024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
662024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 3 reason = Timeout
672024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
682024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 5 reason = Timeout
692024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 3 reason = Timeout
702024-08-26T20:30:20.247ZINFOnexus (BackgroundTasks): Abandoned VMM reaper running background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:76
712024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
722024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
732024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 5 reason = Timeout
742024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
752024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 5 reason = Timeout
762024-08-26T20:30:20.247ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
772024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
782024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 5 reason = Timeout
792024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
802024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 5 reason = Timeout
812024-08-26T20:30:20.248ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task started background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:417
822024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
832024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
842024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
852024-08-26T20:30:20.248ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("be00167b-db1f-4cc1-86f1-a2af7f1c31f8.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("d2b5c33b-9149-4357-86b1-60e922261ec4.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("d2b5c33b-9149-4357-86b1-60e922261ec4.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 919, tv_nsec: 424102036 } })
862024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
872024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
882024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
892024-08-26T20:30:20.248ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("316f64f2-8fa5-43f3-8d5d-92212e3046c4.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("35cb7d67-b9f1-4dcf-bac3-28244fb25d88.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("e37633b8-b58b-40c2-b3c2-6873358565b7.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("e37633b8-b58b-40c2-b3c2-6873358565b7.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:3::1))) }], valid_until: Instant { tv_sec: 919, tv_nsec: 424370729 } })
902024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
912024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
922024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
932024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
942024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
952024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
962024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
972024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
982024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
992024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
1002024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
1012024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
1022024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
1032024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
1042024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
1052024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
1062024-08-26T20:30:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
1072024-08-26T20:30:20.249ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1082024-08-26T20:30:20.249ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
1092024-08-26T20:30:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
1102024-08-26T20:30:20.250ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1112024-08-26T20:30:20.250ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1122024-08-26T20:30:20.251ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1132024-08-26T20:30:20.251ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
1142024-08-26T20:30:20.251ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 3.968928ms iteration = 3 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
1152024-08-26T20:30:20.251ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.967177ms iteration = 5 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
1162024-08-26T20:30:20.251ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
1172024-08-26T20:30:20.251ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
1182024-08-26T20:30:20.251ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
1192024-08-26T20:30:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
1202024-08-26T20:30:20.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1212024-08-26T20:30:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
1222024-08-26T20:30:20.252ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 919, tv_nsec: 428067423 } })
1232024-08-26T20:30:20.252ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } dns_group = internal
1242024-08-26T20:30:20.252ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 4.776754ms iteration = 3 status = {"addresses":["[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353","[fd00:1122:3344:3::1]:5353"]}
1252024-08-26T20:30:20.252ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1262024-08-26T20:30:20.252ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1272024-08-26T20:30:20.253ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1282024-08-26T20:30:20.253ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1050
1292024-08-26T20:30:20.254ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
1302024-08-26T20:30:20.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
1312024-08-26T20:30:20.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
1322024-08-26T20:30:20.254ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
1332024-08-26T20:30:20.254ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
1342024-08-26T20:30:20.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
1352024-08-26T20:30:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
1362024-08-26T20:30:20.255ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1372024-08-26T20:30:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
1382024-08-26T20:30:20.255ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
1392024-08-26T20:30:20.255ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1402024-08-26T20:30:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
1412024-08-26T20:30:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
1422024-08-26T20:30:20.256ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
1432024-08-26T20:30:20.256ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 8.63743ms iteration = 5 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
1442024-08-26T20:30:20.256ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 8.589101ms iteration = 5 status = {"errors":[],"garbage_collect_requested":[]}
1452024-08-26T20:30:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
1462024-08-26T20:30:20.256ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
1472024-08-26T20:30:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
1482024-08-26T20:30:20.256ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
1492024-08-26T20:30:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
1502024-08-26T20:30:20.256ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
1512024-08-26T20:30:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
1522024-08-26T20:30:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
1532024-08-26T20:30:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
1542024-08-26T20:30:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
1552024-08-26T20:30:20.257ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
1562024-08-26T20:30:20.257ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1572024-08-26T20:30:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
1582024-08-26T20:30:20.257ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 9.929603ms iteration = 3 status = {"errors":[],"found_port_ok":[]}
1592024-08-26T20:30:20.258ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-08-26 20:20:20.247547743 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
1602024-08-26T20:30:20.258ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 10.493538ms iteration = 3 status = {"expiration":"2024-08-26T20:20:20.247547743Z","pruned":[]}
1612024-08-26T20:30:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
1622024-08-26T20:30:20.258ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1632024-08-26T20:30:20.258ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 11.295506ms iteration = 13 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
1642024-08-26T20:30:20.258ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
1652024-08-26T20:30:20.258ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
1662024-08-26T20:30:20.258ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
1672024-08-26T20:30:20.258ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
1682024-08-26T20:30:20.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
1692024-08-26T20:30:20.259ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 919, tv_nsec: 435477913 } })
1702024-08-26T20:30:20.259ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
1712024-08-26T20:30:20.259ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
1722024-08-26T20:30:20.260ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
1732024-08-26T20:30:20.260ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
1742024-08-26T20:30:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
1752024-08-26T20:30:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
1762024-08-26T20:30:20.260ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
1772024-08-26T20:30:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
1782024-08-26T20:30:20.260ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
1792024-08-26T20:30:20.260ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
1802024-08-26T20:30:20.260ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
1812024-08-26T20:30:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
1822024-08-26T20:30:20.260ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1832024-08-26T20:30:20.260ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
1842024-08-26T20:30:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
1852024-08-26T20:30:20.260ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1050
1862024-08-26T20:30:20.260ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 919, tv_nsec: 436772605 } })
1872024-08-26T20:30:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
1882024-08-26T20:30:20.261ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
1892024-08-26T20:30:20.261ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1050 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
1902024-08-26T20:30:20.261ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
1912024-08-26T20:30:20.261ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
1922024-08-26T20:30:20.261ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
1932024-08-26T20:30:20.262ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
1942024-08-26T20:30:20.262ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:220 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
1952024-08-26T20:30:20.262ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 14.322624ms iteration = 5 status = {"destroyed_active_vmms":0,"error":null,"saga_start_failures":0,"sagas_completed":0,"sagas_failed":0,"sagas_started":0,"terminated_active_migrations":0}
1962024-08-26T20:30:20.262ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
1972024-08-26T20:30:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
1982024-08-26T20:30:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
1992024-08-26T20:30:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
2002024-08-26T20:30:20.263ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
2012024-08-26T20:30:20.263ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
2022024-08-26T20:30:20.263ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
2032024-08-26T20:30:20.263ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
2042024-08-26T20:30:20.263ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
2052024-08-26T20:30:20.263ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
2062024-08-26T20:30:20.263ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
2072024-08-26T20:30:20.263ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
2082024-08-26T20:30:20.263ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
2092024-08-26T20:30:20.264ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
2102024-08-26T20:30:20.264ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task done background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:439
2112024-08-26T20:30:20.264ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 17.01623ms iteration = 5 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
2122024-08-26T20:30:20.266ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:454 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
2132024-08-26T20:30:20.266ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 18.661925ms iteration = 5 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
2142024-08-26T20:30:20.267ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
2152024-08-26T20:30:20.267ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
2162024-08-26T20:30:20.267ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 20.577473ms iteration = 13 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
2172024-08-26T20:30:20.268ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
2182024-08-26T20:30:20.268ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
2192024-08-26T20:30:20.268ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
2202024-08-26T20:30:20.269ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
2212024-08-26T20:30:20.270ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
2222024-08-26T20:30:20.270ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
2232024-08-26T20:30:20.270ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
2242024-08-26T20:30:20.270ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0
2252024-08-26T20:30:20.270ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:189 found = 0 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0
2262024-08-26T20:30:20.270ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 23.090786ms iteration = 3 status = {"error":null,"error_count":0,"found":0,"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0}
2272024-08-26T20:30:20.275ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 27.85428ms iteration = 6 status = {"physical_disks_added":0}
2282024-08-26T20:30:20.287ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
2292024-08-26T20:30:20.288ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
2302024-08-26T20:30:20.290ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
2312024-08-26T20:30:20.290ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
2322024-08-26T20:30:20.290ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
2332024-08-26T20:30:20.290ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
2342024-08-26T20:30:20.291ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
2352024-08-26T20:30:20.291ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 44.128363ms iteration = 5 status = {"region_replacement_started_err":0,"region_replacement_started_ok":0}
2362024-08-26T20:30:20.293ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: c9d3ee27-fe3f-422b-93ce-695a8356d9d2, lookup_type: ById(c9d3ee27-fe3f-422b-93ce-695a8356d9d2) } result = Ok(())
2372024-08-26T20:30:20.293ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
2382024-08-26T20:30:20.293ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
2392024-08-26T20:30:20.293ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
2402024-08-26T20:30:20.293ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
2412024-08-26T20:30:20.293ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 46.410194ms iteration = 3 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
2422024-08-26T20:30:20.294ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 47.233412ms iteration = 5 status = 0
2432024-08-26T20:30:20.296ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
2442024-08-26T20:30:20.296ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
2452024-08-26T20:30:20.297ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
2462024-08-26T20:30:20.297ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
2472024-08-26T20:30:20.297ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
2482024-08-26T20:30:20.297ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
2492024-08-26T20:30:20.297ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
2502024-08-26T20:30:20.298ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
2512024-08-26T20:30:20.300ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
2522024-08-26T20:30:20.300ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
2532024-08-26T20:30:20.300ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
2542024-08-26T20:30:20.302ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
2552024-08-26T20:30:20.302ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = d840a78c-2f0d-4cdf-9962-31a3dae384a2 dns_zone_name = oxide.test found_so_far = 0 version = 2
2562024-08-26T20:30:20.302ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
2572024-08-26T20:30:20.302ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
2582024-08-26T20:30:20.303ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
2592024-08-26T20:30:20.303ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
2602024-08-26T20:30:20.303ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
2612024-08-26T20:30:20.303ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
2622024-08-26T20:30:20.305ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
2632024-08-26T20:30:20.305ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
2642024-08-26T20:30:20.306ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
2652024-08-26T20:30:20.306ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
2662024-08-26T20:30:20.306ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
2672024-08-26T20:30:20.306ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
2682024-08-26T20:30:20.306ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
2692024-08-26T20:30:20.307ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = d840a78c-2f0d-4cdf-9962-31a3dae384a2 dns_zone_name = oxide.test found_so_far = 1 version = 2
2702024-08-26T20:30:20.307ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
2712024-08-26T20:30:20.307ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-08-26 20:29:01.487349 UTC dns_group = external generation = 2
2722024-08-26T20:30:20.307ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 60.024102ms iteration = 3 status = {"generation":2}
2732024-08-26T20:30:20.308ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
2742024-08-26T20:30:20.308ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
2752024-08-26T20:30:20.308ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
2762024-08-26T20:30:20.316ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-08-26T20:28:18.089116Z, time_modified: 2024-08-26T20:28:18.089116Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 80a72585-f38a-4ccb-b69a-caafab415ca2, db_silo: Silo { identity: SiloIdentity { id: 80a72585-f38a-4ccb-b69a-caafab415ca2, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-08-26T20:29:01.450089Z, time_modified: 2024-08-26T20:29:01.450089Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 80a72585-f38a-4ccb-b69a-caafab415ca2, db_silo: Silo { identity: SiloIdentity { id: 80a72585-f38a-4ccb-b69a-caafab415ca2, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-08-26T20:29:01.450089Z, time_modified: 2024-08-26T20:29:01.450089Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923" }] }) }
2772024-08-26T20:30:20.316ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 69.277268ms iteration = 3 status = {"by_dns_name":{"default-silo.sys.oxide.test":{"silo_id":"001de000-5110-4000-8000-000000000000","tls_certs":[]},"recovery.sys.oxide.test":{"silo_id":"80a72585-f38a-4ccb-b69a-caafab415ca2","tls_certs":["81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923"]}},"default_endpoint":{"silo_id":"80a72585-f38a-4ccb-b69a-caafab415ca2","tls_certs":["81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923"]},"warnings":["silo 001de000-5110-4000-8000-000000000000 with DNS name \\"default-silo.sys.oxide.test\\" has no usable certificates"]}
2782024-08-26T20:30:20.424ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
2792024-08-26T20:30:20.445ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1058 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 zone_address = fd00:1122:3344:101::2
2802024-08-26T20:30:20.445ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
2812024-08-26T20:30:20.447ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "6c87ec48-9cdf-4281-92e4-8b3ffe0d074e", "content-length": "26", "date": "Mon, 26 Aug 2024 20:30:20 GMT"} })
2822024-08-26T20:30:20.447ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1063 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
2832024-08-26T20:30:20.449ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
2842024-08-26T20:30:20.503ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
2852024-08-26T20:30:20.503ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/v2p", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "8b817f08-b83d-4b64-a82a-e7cc76082b34", "content-length": "691", "date": "Mon, 26 Aug 2024 20:30:19 GMT"} })
2862024-08-26T20:30:20.503ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = sock
2872024-08-26T20:30:20.503ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = sock
2882024-08-26T20:30:20.503ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = sock
2892024-08-26T20:30:20.505ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 257.706211ms iteration = 5 status = {}
2902024-08-26T20:30:20.580ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
2912024-08-26T20:30:20.607ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
2922024-08-26T20:30:20.607ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
2932024-08-26T20:30:20.608ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "947fb0e4-8b9b-418d-9e9d-9a1b3218c573", "content-length": "26", "date": "Mon, 26 Aug 2024 20:30:20 GMT"} })
2942024-08-26T20:30:20.608ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
2952024-08-26T20:30:20.610ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
2962024-08-26T20:30:20.615ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
2972024-08-26T20:30:20.615ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::8)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "df5bb5e3-178c-4aa2-876b-7580417c95d4", "date": "Mon, 26 Aug 2024 20:30:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
2982024-08-26T20:30:20.615ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
2992024-08-26T20:30:20.617ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::9)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "6bcc7b91-6a4d-40cb-afed-c2763b1f633e", "date": "Mon, 26 Aug 2024 20:30:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
3002024-08-26T20:30:20.617ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
3012024-08-26T20:30:20.618ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
3022024-08-26T20:30:20.618ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 371.466924ms iteration = 5 status = {"generation":2,"server_results":{"[fd00:1122:3344:101::8]:5353":{"Ok":null},"[fd00:1122:3344:101::9]:5353":{"Ok":null}}}
3032024-08-26T20:30:20.619ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
3042024-08-26T20:30:20.619ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
3052024-08-26T20:30:20.619ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "baa40ee7-4b30-4009-8669-15387b688b3b", "content-length": "26", "date": "Mon, 26 Aug 2024 20:30:20 GMT"} })
3062024-08-26T20:30:20.619ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
3072024-08-26T20:30:20.620ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
3082024-08-26T20:30:20.621ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "9cbb2c26-93f2-48a5-b1aa-ac6948e5258b", "content-length": "372", "date": "Mon, 26 Aug 2024 20:30:19 GMT"} })
3092024-08-26T20:30:20.621ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
3102024-08-26T20:30:20.621ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
3112024-08-26T20:30:20.622ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
3122024-08-26T20:30:20.622ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
3132024-08-26T20:30:20.623ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
3142024-08-26T20:30:20.623ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = b82a4171-23b2-444b-9597-09085a7a2fb8 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
3152024-08-26T20:30:20.623ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
3162024-08-26T20:30:20.623ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
3172024-08-26T20:30:20.623ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
3182024-08-26T20:30:20.624ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
3192024-08-26T20:30:20.625ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
3202024-08-26T20:30:20.625ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
3212024-08-26T20:30:20.627ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
3222024-08-26T20:30:20.627ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
3232024-08-26T20:30:20.627ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
3242024-08-26T20:30:20.627ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
3252024-08-26T20:30:20.628ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 204, headers: {"x-request-id": "bb0a235a-63c6-4ee8-bd14-d42742bd5ff0", "date": "Mon, 26 Aug 2024 20:30:19 GMT"} })
3262024-08-26T20:30:20.628ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = b82a4171-23b2-444b-9597-09085a7a2fb8 dns_zone_name = control-plane.oxide.internal found_so_far = 50 version = 1
3272024-08-26T20:30:20.628ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
3282024-08-26T20:30:20.628ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-08-26 20:29:01.089496 UTC dns_group = internal generation = 1
3292024-08-26T20:30:20.628ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 381.391326ms iteration = 3 status = {"generation":1}
3302024-08-26T20:30:20.629ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 381.964472ms iteration = 5 status = {}
3312024-08-26T20:30:20.714ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
3322024-08-26T20:30:20.721ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 474.004843ms iteration = 5 status = {}
3332024-08-26T20:30:20.729ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
3342024-08-26T20:30:20.730ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "38ae8bfc-c1d2-44ee-bbd0-adc1cdb20dee", "content-length": "1", "date": "Mon, 26 Aug 2024 20:30:20 GMT"} })
3352024-08-26T20:30:20.733ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
3362024-08-26T20:30:20.733ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
3372024-08-26T20:30:20.734ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 486.992434ms iteration = 5 status = null
3382024-08-26T20:30:20.783ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
3392024-08-26T20:30:20.784ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:2::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "6b0f4bb3-ae34-4329-9218-9d723b4351d7", "date": "Mon, 26 Aug 2024 20:30:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
3402024-08-26T20:30:20.784ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
3412024-08-26T20:30:20.785ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:1::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "2e330a1e-0991-4ef7-a460-2ed55ce61639", "date": "Mon, 26 Aug 2024 20:30:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
3422024-08-26T20:30:20.785ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
3432024-08-26T20:30:20.787ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:3::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "383b7a20-059a-4678-a248-76ea2a8f710f", "date": "Mon, 26 Aug 2024 20:30:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
3442024-08-26T20:30:20.787ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
3452024-08-26T20:30:20.788ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
3462024-08-26T20:30:20.788ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 541.04397ms iteration = 5 status = {"generation":1,"server_results":{"[fd00:1122:3344:1::1]:5353":{"Ok":null},"[fd00:1122:3344:2::1]:5353":{"Ok":null},"[fd00:1122:3344:3::1]:5353":{"Ok":null}}}
3472024-08-26T20:30:20.791ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
3482024-08-26T20:30:20.791ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
3492024-08-26T20:30:20.793ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:147 port = SwitchPort { id: 77948b01-64a9-403e-be7b-4ef5e426dda4, rack_id: 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(735476d6-05ca-4fbd-a182-ab082f700355) } rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
3502024-08-26T20:30:20.793ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
3512024-08-26T20:30:20.793ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
3522024-08-26T20:30:20.800ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
3532024-08-26T20:30:20.801ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/port/qsfp0/settings", query: Some("tag=omicron"), fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "92d7aaa5-878a-47e2-bcbf-bb88b24362cf", "content-length": "121", "date": "Mon, 26 Aug 2024 20:30:20 GMT"} })
3542024-08-26T20:30:20.801ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1714 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.2.174], params: LinkCreate { autoneg: false, fec: None, kr: false, lane: Some(LinkId(0)), speed: Speed40G } }} } switch_port_id = "qsfp0"
3552024-08-26T20:30:20.801ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1741 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = 735476d6-05ca-4fbd-a182-ab082f700355
3562024-08-26T20:30:20.801ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:379 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
3572024-08-26T20:30:20.801ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
3582024-08-26T20:30:20.801ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:392 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
3592024-08-26T20:30:20.801ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:398 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
3602024-08-26T20:30:20.801ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:418 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
3612024-08-26T20:30:20.801ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
3622024-08-26T20:30:20.802ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
3632024-08-26T20:30:20.803ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
3642024-08-26T20:30:20.803ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv4", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "ca7d13d3-908c-4f19-ba20-16d7bf9a40f2", "content-length": "2", "date": "Mon, 26 Aug 2024 20:30:20 GMT"} })
3652024-08-26T20:30:20.803ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
3662024-08-26T20:30:20.803ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv6", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "67f2afd8-b8ce-4ad4-a3ac-aae93a883e85", "content-length": "52", "date": "Mon, 26 Aug 2024 20:30:20 GMT"} })
3672024-08-26T20:30:20.803ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0" }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:467 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
3682024-08-26T20:30:20.803ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
3692024-08-26T20:30:20.874ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 195 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = d1ac4717-a7c3-426c-947b-7efc7dd54ba4 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
3702024-08-26T20:30:20.892ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/switch-ports", query: None, fragment: None }, status: 204, headers: {"x-request-id": "5e69badc-2923-4904-aaad-05cfbd178d07", "date": "Mon, 26 Aug 2024 20:30:20 GMT"} })
3712024-08-26T20:30:20.892ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:770 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch = Switch0
3722024-08-26T20:30:20.892ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
3732024-08-26T20:30:20.892ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/network-bootstore-config", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "5ea2c682-adad-4c97-b80f-43c5ceb571bc", "content-length": "667", "date": "Mon, 26 Aug 2024 20:30:20 GMT"} })
3742024-08-26T20:30:20.892ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
3752024-08-26T20:30:20.893ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
3762024-08-26T20:30:20.895ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
3772024-08-26T20:30:20.895ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
3782024-08-26T20:30:20.898ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
3792024-08-26T20:30:20.898ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
3802024-08-26T20:30:20.899ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
3812024-08-26T20:30:20.900ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
3822024-08-26T20:30:20.906ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 659.234976ms iteration = 5 status = {}
3832024-08-26T20:30:30.246ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 14 reason = Timeout
3842024-08-26T20:30:30.247ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
3852024-08-26T20:30:30.247ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
3862024-08-26T20:30:30.247ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 14 reason = Timeout
3872024-08-26T20:30:30.248ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
3882024-08-26T20:30:30.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
3892024-08-26T20:30:30.248ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
3902024-08-26T20:30:30.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
3912024-08-26T20:30:30.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
3922024-08-26T20:30:30.250ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 2.16087ms iteration = 14 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
3932024-08-26T20:30:30.250ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
3942024-08-26T20:30:30.250ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
3952024-08-26T20:30:30.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
3962024-08-26T20:30:30.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
3972024-08-26T20:30:30.252ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
3982024-08-26T20:30:30.252ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
3992024-08-26T20:30:30.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
4002024-08-26T20:30:30.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
4012024-08-26T20:30:30.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
4022024-08-26T20:30:30.262ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
4032024-08-26T20:30:30.262ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 15.494664ms iteration = 14 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
4042024-08-26T20:30:30.874ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 780 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = f2b62960-4775-4f84-a529-c166841c7eee response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
4052024-08-26T20:30:40.247ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 15 reason = Timeout
4062024-08-26T20:30:40.247ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
4072024-08-26T20:30:40.247ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
4082024-08-26T20:30:40.248ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 15 reason = Timeout
4092024-08-26T20:30:40.248ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
4102024-08-26T20:30:40.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
4112024-08-26T20:30:40.248ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
4122024-08-26T20:30:40.250ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
4132024-08-26T20:30:40.250ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
4142024-08-26T20:30:40.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
4152024-08-26T20:30:40.252ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
4162024-08-26T20:30:40.252ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
4172024-08-26T20:30:40.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
4182024-08-26T20:30:40.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
4192024-08-26T20:30:40.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
4202024-08-26T20:30:40.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
4212024-08-26T20:30:40.261ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
4222024-08-26T20:30:40.261ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 13.881ms iteration = 15 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
4232024-08-26T20:30:40.274ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
4242024-08-26T20:30:40.274ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
4252024-08-26T20:30:40.275ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 27.160414ms iteration = 15 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
4262024-08-26T20:30:40.874ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 223 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = c4ffaf43-9ab0-4b46-a988-5f103da3bd51 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
4272024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 16 reason = Timeout
4282024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 6 reason = Timeout
4292024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
4302024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 6 reason = Timeout
4312024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
4322024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
4332024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 6 reason = Timeout
4342024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
4352024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 6 reason = Timeout
4362024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 7 reason = Timeout
4372024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 6 reason = Timeout
4382024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 6 reason = Timeout
4392024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
4402024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
4412024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 6 reason = Timeout
4422024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
4432024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
4442024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
4452024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
4462024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
4472024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
4482024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
4492024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
4502024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
4512024-08-26T20:30:50.247ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
4522024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 6 reason = Timeout
4532024-08-26T20:30:50.248ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task started background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:417
4542024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 6 reason = Timeout
4552024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
4562024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
4572024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 6 reason = Timeout
4582024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 6 reason = Timeout
4592024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
4602024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
4612024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
4622024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 6 reason = Timeout
4632024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
4642024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 16 reason = Timeout
4652024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
4662024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 6 reason = Timeout
4672024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
4682024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
4692024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
4702024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
4712024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
4722024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
4732024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
4742024-08-26T20:30:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
4752024-08-26T20:30:50.249ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
4762024-08-26T20:30:50.249ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
4772024-08-26T20:30:50.250ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
4782024-08-26T20:30:50.250ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.383042ms iteration = 6 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
4792024-08-26T20:30:50.250ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
4802024-08-26T20:30:50.251ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 949, tv_nsec: 426824912 } })
4812024-08-26T20:30:50.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
4822024-08-26T20:30:50.251ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
4832024-08-26T20:30:50.251ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 3.150279ms iteration = 6 status = {"errors":[],"garbage_collect_requested":[]}
4842024-08-26T20:30:50.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
4852024-08-26T20:30:50.251ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 949, tv_nsec: 427638439 } })
4862024-08-26T20:30:50.251ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
4872024-08-26T20:30:50.251ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
4882024-08-26T20:30:50.252ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
4892024-08-26T20:30:50.252ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 4.26165ms iteration = 16 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
4902024-08-26T20:30:50.252ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1050
4912024-08-26T20:30:50.252ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1050
4922024-08-26T20:30:50.252ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
4932024-08-26T20:30:50.252ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
4942024-08-26T20:30:50.253ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
4952024-08-26T20:30:50.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
4962024-08-26T20:30:50.253ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
4972024-08-26T20:30:50.253ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
4982024-08-26T20:30:50.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
4992024-08-26T20:30:50.254ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5002024-08-26T20:30:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
5012024-08-26T20:30:50.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
5022024-08-26T20:30:50.254ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
5032024-08-26T20:30:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
5042024-08-26T20:30:50.254ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
5052024-08-26T20:30:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
5062024-08-26T20:30:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
5072024-08-26T20:30:50.255ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
5082024-08-26T20:30:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
5092024-08-26T20:30:50.255ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
5102024-08-26T20:30:50.255ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
5112024-08-26T20:30:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
5122024-08-26T20:30:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
5132024-08-26T20:30:50.255ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
5142024-08-26T20:30:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
5152024-08-26T20:30:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
5162024-08-26T20:30:50.256ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
5172024-08-26T20:30:50.256ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5182024-08-26T20:30:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
5192024-08-26T20:30:50.256ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
5202024-08-26T20:30:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
5212024-08-26T20:30:50.256ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 949, tv_nsec: 432633426 } })
5222024-08-26T20:30:50.257ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5232024-08-26T20:30:50.257ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1050 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
5242024-08-26T20:30:50.257ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
5252024-08-26T20:30:50.257ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
5262024-08-26T20:30:50.257ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
5272024-08-26T20:30:50.257ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
5282024-08-26T20:30:50.257ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 9.345037ms iteration = 6 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
5292024-08-26T20:30:50.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
5302024-08-26T20:30:50.257ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
5312024-08-26T20:30:50.257ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
5322024-08-26T20:30:50.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
5332024-08-26T20:30:50.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
5342024-08-26T20:30:50.258ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
5352024-08-26T20:30:50.258ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:220 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
5362024-08-26T20:30:50.258ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 10.65584ms iteration = 6 status = {"destroyed_active_vmms":0,"error":null,"saga_start_failures":0,"sagas_completed":0,"sagas_failed":0,"sagas_started":0,"terminated_active_migrations":0}
5372024-08-26T20:30:50.259ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
5382024-08-26T20:30:50.259ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
5392024-08-26T20:30:50.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
5402024-08-26T20:30:50.260ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
5412024-08-26T20:30:50.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
5422024-08-26T20:30:50.267ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:454 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
5432024-08-26T20:30:50.267ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 18.779546ms iteration = 6 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
5442024-08-26T20:30:50.276ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
5452024-08-26T20:30:50.276ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 29.075552ms iteration = 16 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
5462024-08-26T20:30:50.276ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
5472024-08-26T20:30:50.276ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
5482024-08-26T20:30:50.277ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5492024-08-26T20:30:50.279ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
5502024-08-26T20:30:50.279ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
5512024-08-26T20:30:50.279ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
5522024-08-26T20:30:50.279ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
5532024-08-26T20:30:50.279ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
5542024-08-26T20:30:50.280ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5552024-08-26T20:30:50.281ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
5562024-08-26T20:30:50.281ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
5572024-08-26T20:30:50.281ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
5582024-08-26T20:30:50.281ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
5592024-08-26T20:30:50.281ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
5602024-08-26T20:30:50.282ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5612024-08-26T20:30:50.284ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: c9d3ee27-fe3f-422b-93ce-695a8356d9d2, lookup_type: ById(c9d3ee27-fe3f-422b-93ce-695a8356d9d2) } result = Ok(())
5622024-08-26T20:30:50.284ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
5632024-08-26T20:30:50.284ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
5642024-08-26T20:30:50.285ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 37.958445ms iteration = 6 status = 0
5652024-08-26T20:30:50.292ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
5662024-08-26T20:30:50.292ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
5672024-08-26T20:30:50.292ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
5682024-08-26T20:30:50.292ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
5692024-08-26T20:30:50.293ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 46.06421ms iteration = 6 status = {"region_replacement_started_err":0,"region_replacement_started_ok":0}
5702024-08-26T20:30:50.293ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task done background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:439
5712024-08-26T20:30:50.293ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 45.256673ms iteration = 6 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
5722024-08-26T20:30:50.296ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 49.669265ms iteration = 7 status = {"physical_disks_added":0}
5732024-08-26T20:30:50.393ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
5742024-08-26T20:30:50.393ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
5752024-08-26T20:30:50.395ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "ac9514be-af41-4d13-9bbe-bcbf69585537", "content-length": "26", "date": "Mon, 26 Aug 2024 20:30:50 GMT"} })
5762024-08-26T20:30:50.395ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
5772024-08-26T20:30:50.397ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
5782024-08-26T20:30:50.399ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
5792024-08-26T20:30:50.400ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "badba51e-4b5b-406c-86eb-b424cee3ec93", "content-length": "372", "date": "Mon, 26 Aug 2024 20:30:49 GMT"} })
5802024-08-26T20:30:50.400ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
5812024-08-26T20:30:50.400ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
5822024-08-26T20:30:50.401ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
5832024-08-26T20:30:50.402ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
5842024-08-26T20:30:50.404ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
5852024-08-26T20:30:50.404ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
5862024-08-26T20:30:50.406ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
5872024-08-26T20:30:50.406ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 204, headers: {"x-request-id": "a59004b9-c559-4ee9-8cc8-6be8796b7b3a", "date": "Mon, 26 Aug 2024 20:30:49 GMT"} })
5882024-08-26T20:30:50.407ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 159.684503ms iteration = 6 status = {}
5892024-08-26T20:30:50.424ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
5902024-08-26T20:30:50.424ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/v2p", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "16385a25-3bad-4b29-965a-64e7699b2cfc", "content-length": "691", "date": "Mon, 26 Aug 2024 20:30:49 GMT"} })
5912024-08-26T20:30:50.424ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = sock
5922024-08-26T20:30:50.424ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = sock
5932024-08-26T20:30:50.424ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = sock
5942024-08-26T20:30:50.426ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 179.188405ms iteration = 6 status = {}
5952024-08-26T20:30:50.489ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
5962024-08-26T20:30:50.489ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
5972024-08-26T20:30:50.489ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "411bf2a2-2f34-4ca5-915e-f74b175ba9dd", "content-length": "26", "date": "Mon, 26 Aug 2024 20:30:50 GMT"} })
5982024-08-26T20:30:50.489ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
5992024-08-26T20:30:50.489ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1058 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 zone_address = fd00:1122:3344:101::2
6002024-08-26T20:30:50.489ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
6012024-08-26T20:30:50.489ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "28158336-750e-4cf6-843e-1a7d98f1df27", "content-length": "26", "date": "Mon, 26 Aug 2024 20:30:50 GMT"} })
6022024-08-26T20:30:50.489ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1063 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
6032024-08-26T20:30:50.492ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
6042024-08-26T20:30:50.492ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 245.91911ms iteration = 6 status = {}
6052024-08-26T20:30:50.493ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
6062024-08-26T20:30:50.552ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
6072024-08-26T20:30:50.553ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "ceb20dfa-796d-43a8-ba07-8d8b545c4645", "content-length": "1", "date": "Mon, 26 Aug 2024 20:30:50 GMT"} })
6082024-08-26T20:30:50.554ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
6092024-08-26T20:30:50.555ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
6102024-08-26T20:30:50.556ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 309.187251ms iteration = 6 status = null
6112024-08-26T20:30:50.615ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
6122024-08-26T20:30:50.615ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
6132024-08-26T20:30:50.618ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:147 port = SwitchPort { id: 77948b01-64a9-403e-be7b-4ef5e426dda4, rack_id: 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(735476d6-05ca-4fbd-a182-ab082f700355) } rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
6142024-08-26T20:30:50.618ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
6152024-08-26T20:30:50.618ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
6162024-08-26T20:30:50.626ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
6172024-08-26T20:30:50.626ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/port/qsfp0/settings", query: Some("tag=omicron"), fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "497c899d-457a-463b-a335-f3d72775fe0d", "content-length": "121", "date": "Mon, 26 Aug 2024 20:30:50 GMT"} })
6182024-08-26T20:30:50.626ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1714 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.2.174], params: LinkCreate { autoneg: false, fec: None, kr: false, lane: Some(LinkId(0)), speed: Speed40G } }} } switch_port_id = "qsfp0"
6192024-08-26T20:30:50.626ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1741 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = 735476d6-05ca-4fbd-a182-ab082f700355
6202024-08-26T20:30:50.627ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:379 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
6212024-08-26T20:30:50.627ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
6222024-08-26T20:30:50.627ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:392 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
6232024-08-26T20:30:50.627ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:398 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
6242024-08-26T20:30:50.627ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:418 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
6252024-08-26T20:30:50.627ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
6262024-08-26T20:30:50.627ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
6272024-08-26T20:30:50.628ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
6282024-08-26T20:30:50.629ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv4", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "c5752ae5-1278-499a-87a9-47861697179a", "content-length": "2", "date": "Mon, 26 Aug 2024 20:30:50 GMT"} })
6292024-08-26T20:30:50.629ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
6302024-08-26T20:30:50.629ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv6", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "e58480a5-4d71-40f0-87a5-95ec50e5ee46", "content-length": "52", "date": "Mon, 26 Aug 2024 20:30:50 GMT"} })
6312024-08-26T20:30:50.629ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0" }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:467 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
6322024-08-26T20:30:50.629ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
6332024-08-26T20:30:50.729ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/switch-ports", query: None, fragment: None }, status: 204, headers: {"x-request-id": "1125a313-d19f-450e-8f85-8c85aadb9936", "date": "Mon, 26 Aug 2024 20:30:50 GMT"} })
6342024-08-26T20:30:50.729ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:770 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch = Switch0
6352024-08-26T20:30:50.729ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
6362024-08-26T20:30:50.730ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/network-bootstore-config", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "4f5551d7-bca4-4d83-8427-926d909ae6cf", "content-length": "667", "date": "Mon, 26 Aug 2024 20:30:50 GMT"} })
6372024-08-26T20:30:50.730ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
6382024-08-26T20:30:50.730ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
6392024-08-26T20:30:50.733ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
6402024-08-26T20:30:50.733ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
6412024-08-26T20:30:50.736ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
6422024-08-26T20:30:50.736ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
6432024-08-26T20:30:50.738ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
6442024-08-26T20:30:50.738ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
6452024-08-26T20:30:50.744ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 497.739735ms iteration = 6 status = {}
6462024-08-26T20:30:50.874ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 254 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = d5543f5a-0f01-41f7-a858-92014cafc208 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
6472024-08-26T20:31:00.247ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 17 reason = Timeout
6482024-08-26T20:31:00.247ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
6492024-08-26T20:31:00.247ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
6502024-08-26T20:31:00.248ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 17 reason = Timeout
6512024-08-26T20:31:00.248ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
6522024-08-26T20:31:00.248ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
6532024-08-26T20:31:00.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
6542024-08-26T20:31:00.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
6552024-08-26T20:31:00.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
6562024-08-26T20:31:00.251ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
6572024-08-26T20:31:00.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
6582024-08-26T20:31:00.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
6592024-08-26T20:31:00.252ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
6602024-08-26T20:31:00.252ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
6612024-08-26T20:31:00.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
6622024-08-26T20:31:00.255ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
6632024-08-26T20:31:00.255ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
6642024-08-26T20:31:00.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
6652024-08-26T20:31:00.261ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
6662024-08-26T20:31:00.261ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 14.687407ms iteration = 17 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
6672024-08-26T20:31:00.275ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 26.616238ms iteration = 17 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
6682024-08-26T20:31:00.874ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 227 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = d86a9ead-e76c-4c83-8ec4-321cf0a70d50 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
6692024-08-26T20:31:10.247ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 18 reason = Timeout
6702024-08-26T20:31:10.247ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
6712024-08-26T20:31:10.247ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
6722024-08-26T20:31:10.248ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 18 reason = Timeout
6732024-08-26T20:31:10.248ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
6742024-08-26T20:31:10.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
6752024-08-26T20:31:10.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
6762024-08-26T20:31:10.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
6772024-08-26T20:31:10.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
6782024-08-26T20:31:10.250ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 2.530344ms iteration = 18 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
6792024-08-26T20:31:10.251ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
6802024-08-26T20:31:10.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
6812024-08-26T20:31:10.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
6822024-08-26T20:31:10.252ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
6832024-08-26T20:31:10.252ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
6842024-08-26T20:31:10.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
6852024-08-26T20:31:10.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
6862024-08-26T20:31:10.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
6872024-08-26T20:31:10.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
6882024-08-26T20:31:10.262ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
6892024-08-26T20:31:10.262ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 15.750147ms iteration = 18 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
6902024-08-26T20:31:10.875ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 235 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 40a092e5-01db-40dd-8ab0-811c4ac2ef8a response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
6912024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 6 reason = Timeout
6922024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 4 reason = Timeout
6932024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 4 reason = Timeout
6942024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 7 reason = Timeout
6952024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
6962024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
6972024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 7 reason = Timeout
6982024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 4 reason = Timeout
6992024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 4 reason = Timeout
7002024-08-26T20:31:20.247ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
7012024-08-26T20:31:20.247ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-08-26 20:21:20.247593318 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
7022024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = crdb_node_id_collector iteration = 3 reason = Timeout
7032024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
7042024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
7052024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 7 reason = Timeout
7062024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
7072024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
7082024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 7 reason = Timeout
7092024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
7102024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 7 reason = Timeout
7112024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 8 reason = Timeout
7122024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 19 reason = Timeout
7132024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
7142024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 5 reason = Timeout
7152024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 7 reason = Timeout
7162024-08-26T20:31:20.247ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:80 reason = blueprint disabled target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460
7172024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
7182024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
7192024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 9.58\u{b5}s iteration = 5 status = {"error":"blueprint disabled","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460"}
7202024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
7212024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 4 reason = Timeout
7222024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
7232024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
7242024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 4 reason = Timeout
7252024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 7 reason = Timeout
7262024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
7272024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
7282024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
7292024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 4 reason = Timeout
7302024-08-26T20:31:20.247ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 6 reason = Timeout
7312024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
7322024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 7 reason = Timeout
7332024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
7342024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
7352024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 7 reason = Timeout
7362024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
7372024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
7382024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 4 reason = Timeout
7392024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 7 reason = Timeout
7402024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 19 reason = Timeout
7412024-08-26T20:31:20.248ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task started background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:417
7422024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
7432024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
7442024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
7452024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 4 reason = Timeout
7462024-08-26T20:31:20.248ZINFOnexus (BackgroundTasks): Abandoned VMM reaper running background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:76
7472024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
7482024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 7 reason = Timeout
7492024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 7 reason = Timeout
7502024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
7512024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
7522024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 7 reason = Timeout
7532024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
7542024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
7552024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
7562024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
7572024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
7582024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
7592024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
7602024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
7612024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
7622024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
7632024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
7642024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
7652024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
7662024-08-26T20:31:20.248ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("be00167b-db1f-4cc1-86f1-a2af7f1c31f8.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("d2b5c33b-9149-4357-86b1-60e922261ec4.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("d2b5c33b-9149-4357-86b1-60e922261ec4.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 979, tv_nsec: 423800503 } })
7672024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
7682024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
7692024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
7702024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
7712024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
7722024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
7732024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
7742024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
7752024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
7762024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
7772024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
7782024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
7792024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
7802024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
7812024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
7822024-08-26T20:31:20.248ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
7832024-08-26T20:31:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
7842024-08-26T20:31:20.249ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
7852024-08-26T20:31:20.249ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("316f64f2-8fa5-43f3-8d5d-92212e3046c4.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("35cb7d67-b9f1-4dcf-bac3-28244fb25d88.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("e37633b8-b58b-40c2-b3c2-6873358565b7.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("e37633b8-b58b-40c2-b3c2-6873358565b7.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:3::1))) }], valid_until: Instant { tv_sec: 979, tv_nsec: 424877473 } })
7862024-08-26T20:31:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
7872024-08-26T20:31:20.250ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
7882024-08-26T20:31:20.250ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 2.739456ms iteration = 4 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
7892024-08-26T20:31:20.250ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 979, tv_nsec: 425841782 } })
7902024-08-26T20:31:20.250ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7912024-08-26T20:31:20.250ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7922024-08-26T20:31:20.251ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7932024-08-26T20:31:20.251ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.820505ms iteration = 7 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
7942024-08-26T20:31:20.252ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7952024-08-26T20:31:20.252ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7962024-08-26T20:31:20.252ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-08-26 20:21:20.247593318 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
7972024-08-26T20:31:20.252ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 4.752774ms iteration = 4 status = {"expiration":"2024-08-26T20:21:20.247593318Z","pruned":[]}
7982024-08-26T20:31:20.252ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } dns_group = internal
7992024-08-26T20:31:20.252ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 4.589393ms iteration = 4 status = {"addresses":["[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353","[fd00:1122:3344:3::1]:5353"]}
8002024-08-26T20:31:20.252ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1050
8012024-08-26T20:31:20.252ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 979, tv_nsec: 427966422 } })
8022024-08-26T20:31:20.252ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 4.832266ms iteration = 7 status = {"errors":[],"garbage_collect_requested":[]}
8032024-08-26T20:31:20.253ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1050
8042024-08-26T20:31:20.253ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 5.543581ms iteration = 4 status = {"errors":[],"found_port_ok":[]}
8052024-08-26T20:31:20.253ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8062024-08-26T20:31:20.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
8072024-08-26T20:31:20.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
8082024-08-26T20:31:20.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
8092024-08-26T20:31:20.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
8102024-08-26T20:31:20.254ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
8112024-08-26T20:31:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
8122024-08-26T20:31:20.255ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
8132024-08-26T20:31:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
8142024-08-26T20:31:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
8152024-08-26T20:31:20.255ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
8162024-08-26T20:31:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
8172024-08-26T20:31:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
8182024-08-26T20:31:20.255ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
8192024-08-26T20:31:20.255ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
8202024-08-26T20:31:20.255ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8212024-08-26T20:31:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
8222024-08-26T20:31:20.256ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
8232024-08-26T20:31:20.256ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
8242024-08-26T20:31:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
8252024-08-26T20:31:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
8262024-08-26T20:31:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
8272024-08-26T20:31:20.256ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
8282024-08-26T20:31:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
8292024-08-26T20:31:20.257ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8302024-08-26T20:31:20.257ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8312024-08-26T20:31:20.257ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
8322024-08-26T20:31:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
8332024-08-26T20:31:20.258ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
8342024-08-26T20:31:20.258ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
8352024-08-26T20:31:20.258ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 10.862382ms iteration = 4 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
8362024-08-26T20:31:20.258ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8372024-08-26T20:31:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
8382024-08-26T20:31:20.258ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
8392024-08-26T20:31:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
8402024-08-26T20:31:20.259ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8412024-08-26T20:31:20.259ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
8422024-08-26T20:31:20.259ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
8432024-08-26T20:31:20.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
8442024-08-26T20:31:20.260ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
8452024-08-26T20:31:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
8462024-08-26T20:31:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
8472024-08-26T20:31:20.260ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8482024-08-26T20:31:20.260ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0
8492024-08-26T20:31:20.260ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:189 found = 0 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0
8502024-08-26T20:31:20.261ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 12.88437ms iteration = 4 status = {"error":null,"error_count":0,"found":0,"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0}
8512024-08-26T20:31:20.261ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
8522024-08-26T20:31:20.261ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
8532024-08-26T20:31:20.261ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
8542024-08-26T20:31:20.261ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
8552024-08-26T20:31:20.261ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
8562024-08-26T20:31:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
8572024-08-26T20:31:20.262ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
8582024-08-26T20:31:20.262ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
8592024-08-26T20:31:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
8602024-08-26T20:31:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
8612024-08-26T20:31:20.262ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
8622024-08-26T20:31:20.263ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
8632024-08-26T20:31:20.263ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
8642024-08-26T20:31:20.263ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
8652024-08-26T20:31:20.263ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 979, tv_nsec: 438650332 } })
8662024-08-26T20:31:20.263ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
8672024-08-26T20:31:20.263ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1050 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
8682024-08-26T20:31:20.263ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
8692024-08-26T20:31:20.263ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
8702024-08-26T20:31:20.264ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
8712024-08-26T20:31:20.264ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8722024-08-26T20:31:20.264ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
8732024-08-26T20:31:20.265ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8742024-08-26T20:31:20.266ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
8752024-08-26T20:31:20.266ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
8762024-08-26T20:31:20.266ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
8772024-08-26T20:31:20.269ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
8782024-08-26T20:31:20.269ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
8792024-08-26T20:31:20.269ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
8802024-08-26T20:31:20.280ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:454 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
8812024-08-26T20:31:20.280ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 32.650426ms iteration = 7 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
8822024-08-26T20:31:20.282ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8832024-08-26T20:31:20.283ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
8842024-08-26T20:31:20.283ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
8852024-08-26T20:31:20.283ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
8862024-08-26T20:31:20.284ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8872024-08-26T20:31:20.286ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
8882024-08-26T20:31:20.286ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
8892024-08-26T20:31:20.286ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
8902024-08-26T20:31:20.292ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
8912024-08-26T20:31:20.292ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
8922024-08-26T20:31:20.294ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 46.334203ms iteration = 7 status = {"region_replacement_started_err":0,"region_replacement_started_ok":0}
8932024-08-26T20:31:20.295ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8942024-08-26T20:31:20.296ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
8952024-08-26T20:31:20.296ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
8962024-08-26T20:31:20.296ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
8972024-08-26T20:31:20.296ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
8982024-08-26T20:31:20.297ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
8992024-08-26T20:31:20.297ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
9002024-08-26T20:31:20.297ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
9012024-08-26T20:31:20.298ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
9022024-08-26T20:31:20.298ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
9032024-08-26T20:31:20.298ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
9042024-08-26T20:31:20.298ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
9052024-08-26T20:31:20.298ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
9062024-08-26T20:31:20.299ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task done background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:439
9072024-08-26T20:31:20.299ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 50.998746ms iteration = 7 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
9082024-08-26T20:31:20.299ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
9092024-08-26T20:31:20.299ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
9102024-08-26T20:31:20.299ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
9112024-08-26T20:31:20.299ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
9122024-08-26T20:31:20.299ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
9132024-08-26T20:31:20.299ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9142024-08-26T20:31:20.300ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9152024-08-26T20:31:20.300ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9162024-08-26T20:31:20.302ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
9172024-08-26T20:31:20.302ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
9182024-08-26T20:31:20.302ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
9192024-08-26T20:31:20.303ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9202024-08-26T20:31:20.303ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
9212024-08-26T20:31:20.303ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
9222024-08-26T20:31:20.303ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 55.307937ms iteration = 7 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
9232024-08-26T20:31:20.303ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
9242024-08-26T20:31:20.303ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
9252024-08-26T20:31:20.303ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
9262024-08-26T20:31:20.303ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
9272024-08-26T20:31:20.305ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
9282024-08-26T20:31:20.305ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
9292024-08-26T20:31:20.305ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
9302024-08-26T20:31:20.305ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
9312024-08-26T20:31:20.305ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = d840a78c-2f0d-4cdf-9962-31a3dae384a2 dns_zone_name = oxide.test found_so_far = 0 version = 2
9322024-08-26T20:31:20.305ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
9332024-08-26T20:31:20.305ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
9342024-08-26T20:31:20.306ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
9352024-08-26T20:31:20.306ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
9362024-08-26T20:31:20.306ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
9372024-08-26T20:31:20.306ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9382024-08-26T20:31:20.306ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
9392024-08-26T20:31:20.307ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9402024-08-26T20:31:20.307ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9412024-08-26T20:31:20.309ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
9422024-08-26T20:31:20.309ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
9432024-08-26T20:31:20.309ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
9442024-08-26T20:31:20.309ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
9452024-08-26T20:31:20.309ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
9462024-08-26T20:31:20.309ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
9472024-08-26T20:31:20.309ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 61.865678ms iteration = 19 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
9482024-08-26T20:31:20.309ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
9492024-08-26T20:31:20.309ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
9502024-08-26T20:31:20.309ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
9512024-08-26T20:31:20.309ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
9522024-08-26T20:31:20.309ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
9532024-08-26T20:31:20.309ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
9542024-08-26T20:31:20.310ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9552024-08-26T20:31:20.310ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = d840a78c-2f0d-4cdf-9962-31a3dae384a2 dns_zone_name = oxide.test found_so_far = 1 version = 2
9562024-08-26T20:31:20.310ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
9572024-08-26T20:31:20.310ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-08-26 20:29:01.487349 UTC dns_group = external generation = 2
9582024-08-26T20:31:20.310ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 63.252032ms iteration = 4 status = {"generation":2}
9592024-08-26T20:31:20.311ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
9602024-08-26T20:31:20.311ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
9612024-08-26T20:31:20.312ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
9622024-08-26T20:31:20.312ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
9632024-08-26T20:31:20.312ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9642024-08-26T20:31:20.312ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
9652024-08-26T20:31:20.313ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 65.849736ms iteration = 8 status = {"physical_disks_added":0}
9662024-08-26T20:31:20.314ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: c9d3ee27-fe3f-422b-93ce-695a8356d9d2, lookup_type: ById(c9d3ee27-fe3f-422b-93ce-695a8356d9d2) } result = Ok(())
9672024-08-26T20:31:20.314ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
9682024-08-26T20:31:20.315ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
9692024-08-26T20:31:20.315ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 68.327539ms iteration = 7 status = 0
9702024-08-26T20:31:20.324ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-08-26T20:28:18.089116Z, time_modified: 2024-08-26T20:28:18.089116Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 80a72585-f38a-4ccb-b69a-caafab415ca2, db_silo: Silo { identity: SiloIdentity { id: 80a72585-f38a-4ccb-b69a-caafab415ca2, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-08-26T20:29:01.450089Z, time_modified: 2024-08-26T20:29:01.450089Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 80a72585-f38a-4ccb-b69a-caafab415ca2, db_silo: Silo { identity: SiloIdentity { id: 80a72585-f38a-4ccb-b69a-caafab415ca2, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-08-26T20:29:01.450089Z, time_modified: 2024-08-26T20:29:01.450089Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923" }] }) }
9712024-08-26T20:31:20.324ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 76.92066ms iteration = 4 status = {"by_dns_name":{"default-silo.sys.oxide.test":{"silo_id":"001de000-5110-4000-8000-000000000000","tls_certs":[]},"recovery.sys.oxide.test":{"silo_id":"80a72585-f38a-4ccb-b69a-caafab415ca2","tls_certs":["81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923"]}},"default_endpoint":{"silo_id":"80a72585-f38a-4ccb-b69a-caafab415ca2","tls_certs":["81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923"]},"warnings":["silo 001de000-5110-4000-8000-000000000000 with DNS name \\"default-silo.sys.oxide.test\\" has no usable certificates"]}
9722024-08-26T20:31:20.325ZDEBGnexus (BackgroundTasks): activation complete background_task = crdb_node_id_collector elapsed = 77.946509ms iteration = 3 status = {"nsuccess":5}
9732024-08-26T20:31:20.326ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
9742024-08-26T20:31:20.326ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:220 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
9752024-08-26T20:31:20.327ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 78.848437ms iteration = 7 status = {"destroyed_active_vmms":0,"error":null,"saga_start_failures":0,"sagas_completed":0,"sagas_failed":0,"sagas_started":0,"terminated_active_migrations":0}
9762024-08-26T20:31:20.329ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
9772024-08-26T20:31:20.329ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = b82a4171-23b2-444b-9597-09085a7a2fb8 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
9782024-08-26T20:31:20.329ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
9792024-08-26T20:31:20.329ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
9802024-08-26T20:31:20.330ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9812024-08-26T20:31:20.333ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
9822024-08-26T20:31:20.333ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
9832024-08-26T20:31:20.334ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
9842024-08-26T20:31:20.336ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = b82a4171-23b2-444b-9597-09085a7a2fb8 dns_zone_name = control-plane.oxide.internal found_so_far = 50 version = 1
9852024-08-26T20:31:20.336ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
9862024-08-26T20:31:20.336ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-08-26 20:29:01.089496 UTC dns_group = internal generation = 1
9872024-08-26T20:31:20.336ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 88.662189ms iteration = 4 status = {"generation":1}
9882024-08-26T20:31:20.423ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
9892024-08-26T20:31:20.442ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
9902024-08-26T20:31:20.491ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
9912024-08-26T20:31:20.491ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/v2p", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "52ef8872-b30d-4688-81f7-06d40f4fe28e", "content-length": "691", "date": "Mon, 26 Aug 2024 20:31:19 GMT"} })
9922024-08-26T20:31:20.491ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = sock
9932024-08-26T20:31:20.491ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = sock
9942024-08-26T20:31:20.491ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = sock
9952024-08-26T20:31:20.493ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 245.676988ms iteration = 7 status = {}
9962024-08-26T20:31:20.587ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1058 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 zone_address = fd00:1122:3344:101::2
9972024-08-26T20:31:20.587ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
9982024-08-26T20:31:20.588ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "b316e6ce-0796-4687-8e11-f35f6d1f0105", "content-length": "26", "date": "Mon, 26 Aug 2024 20:31:20 GMT"} })
9992024-08-26T20:31:20.588ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1063 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
10002024-08-26T20:31:20.590ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
10012024-08-26T20:31:20.593ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
10022024-08-26T20:31:20.593ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
10032024-08-26T20:31:20.593ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "1db3556b-bae3-4ebe-bc4f-619e33ed1073", "content-length": "26", "date": "Mon, 26 Aug 2024 20:31:20 GMT"} })
10042024-08-26T20:31:20.593ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
10052024-08-26T20:31:20.595ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
10062024-08-26T20:31:20.597ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
10072024-08-26T20:31:20.597ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
10082024-08-26T20:31:20.597ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "ed0cb44b-ce18-486f-a9b7-f313d3bf3d89", "content-length": "26", "date": "Mon, 26 Aug 2024 20:31:20 GMT"} })
10092024-08-26T20:31:20.597ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
10102024-08-26T20:31:20.598ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
10112024-08-26T20:31:20.604ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
10122024-08-26T20:31:20.604ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::8)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "00a3d749-9408-41b6-914c-3ec794a333e5", "date": "Mon, 26 Aug 2024 20:31:19 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
10132024-08-26T20:31:20.604ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
10142024-08-26T20:31:20.606ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::9)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "9d1d79d7-e88b-4ac2-8489-f5323cc0af79", "date": "Mon, 26 Aug 2024 20:31:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
10152024-08-26T20:31:20.606ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
10162024-08-26T20:31:20.607ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
10172024-08-26T20:31:20.607ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 360.36649ms iteration = 6 status = {"generation":2,"server_results":{"[fd00:1122:3344:101::8]:5353":{"Ok":null},"[fd00:1122:3344:101::9]:5353":{"Ok":null}}}
10182024-08-26T20:31:20.623ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
10192024-08-26T20:31:20.634ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
10202024-08-26T20:31:20.635ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 387.708837ms iteration = 19 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
10212024-08-26T20:31:20.635ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "3e991fa2-4604-4863-8721-9ef2fb7c2f79", "content-length": "372", "date": "Mon, 26 Aug 2024 20:31:19 GMT"} })
10222024-08-26T20:31:20.635ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
10232024-08-26T20:31:20.636ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
10242024-08-26T20:31:20.637ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
10252024-08-26T20:31:20.637ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
10262024-08-26T20:31:20.639ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
10272024-08-26T20:31:20.640ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
10282024-08-26T20:31:20.642ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
10292024-08-26T20:31:20.642ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 204, headers: {"x-request-id": "4974cedc-c0f2-4848-b435-9b8388c4ddde", "date": "Mon, 26 Aug 2024 20:31:19 GMT"} })
10302024-08-26T20:31:20.644ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 396.600539ms iteration = 7 status = {}
10312024-08-26T20:31:20.718ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
10322024-08-26T20:31:20.719ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "cde1aa23-34a6-4663-bbdb-ed3ce9bae962", "content-length": "1", "date": "Mon, 26 Aug 2024 20:31:20 GMT"} })
10332024-08-26T20:31:20.724ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
10342024-08-26T20:31:20.724ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 476.591498ms iteration = 7 status = {}
10352024-08-26T20:31:20.724ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
10362024-08-26T20:31:20.725ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 477.952391ms iteration = 7 status = null
10372024-08-26T20:31:20.733ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
10382024-08-26T20:31:20.733ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:3::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "b20fb3e6-1a84-43a2-888f-29552e0a4483", "date": "Mon, 26 Aug 2024 20:31:19 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
10392024-08-26T20:31:20.733ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
10402024-08-26T20:31:20.734ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:2::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "91dbc33f-0fcf-47a4-a990-e07335372a28", "date": "Mon, 26 Aug 2024 20:31:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
10412024-08-26T20:31:20.734ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
10422024-08-26T20:31:20.736ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:1::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "50ee71ee-a696-4e17-a3d7-e59fbc0ad92d", "date": "Mon, 26 Aug 2024 20:31:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
10432024-08-26T20:31:20.736ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
10442024-08-26T20:31:20.737ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
10452024-08-26T20:31:20.737ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 490.146814ms iteration = 6 status = {"generation":1,"server_results":{"[fd00:1122:3344:1::1]:5353":{"Ok":null},"[fd00:1122:3344:2::1]:5353":{"Ok":null},"[fd00:1122:3344:3::1]:5353":{"Ok":null}}}
10462024-08-26T20:31:20.792ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
10472024-08-26T20:31:20.792ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
10482024-08-26T20:31:20.794ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:147 port = SwitchPort { id: 77948b01-64a9-403e-be7b-4ef5e426dda4, rack_id: 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(735476d6-05ca-4fbd-a182-ab082f700355) } rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
10492024-08-26T20:31:20.794ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
10502024-08-26T20:31:20.794ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
10512024-08-26T20:31:20.803ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
10522024-08-26T20:31:20.803ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/port/qsfp0/settings", query: Some("tag=omicron"), fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "254b5e51-088c-4f6c-aede-af90718a2bca", "content-length": "121", "date": "Mon, 26 Aug 2024 20:31:20 GMT"} })
10532024-08-26T20:31:20.803ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1714 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.2.174], params: LinkCreate { autoneg: false, fec: None, kr: false, lane: Some(LinkId(0)), speed: Speed40G } }} } switch_port_id = "qsfp0"
10542024-08-26T20:31:20.803ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1741 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = 735476d6-05ca-4fbd-a182-ab082f700355
10552024-08-26T20:31:20.803ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:379 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
10562024-08-26T20:31:20.803ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
10572024-08-26T20:31:20.804ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:392 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
10582024-08-26T20:31:20.804ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:398 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
10592024-08-26T20:31:20.804ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:418 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
10602024-08-26T20:31:20.804ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
10612024-08-26T20:31:20.804ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
10622024-08-26T20:31:20.805ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
10632024-08-26T20:31:20.805ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv4", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "9dab52f1-9d92-41c9-8633-d52740b2b5d9", "content-length": "2", "date": "Mon, 26 Aug 2024 20:31:20 GMT"} })
10642024-08-26T20:31:20.805ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
10652024-08-26T20:31:20.806ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv6", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "2cce4a12-8ad5-42aa-ae57-1646ba27058e", "content-length": "52", "date": "Mon, 26 Aug 2024 20:31:20 GMT"} })
10662024-08-26T20:31:20.806ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0" }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:467 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
10672024-08-26T20:31:20.806ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
10682024-08-26T20:31:20.875ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 198 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 325b826f-ecae-4dbc-8036-3c42f81f9e7a response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
10692024-08-26T20:31:20.896ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/switch-ports", query: None, fragment: None }, status: 204, headers: {"x-request-id": "fcfa1984-811c-4f62-9c15-4da000bb2eca", "date": "Mon, 26 Aug 2024 20:31:20 GMT"} })
10702024-08-26T20:31:20.896ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:770 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch = Switch0
10712024-08-26T20:31:20.896ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
10722024-08-26T20:31:20.896ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/network-bootstore-config", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "b7b84037-6882-4b26-ae03-c21007119957", "content-length": "667", "date": "Mon, 26 Aug 2024 20:31:20 GMT"} })
10732024-08-26T20:31:20.896ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
10742024-08-26T20:31:20.896ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
10752024-08-26T20:31:20.899ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
10762024-08-26T20:31:20.900ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
10772024-08-26T20:31:20.903ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
10782024-08-26T20:31:20.903ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
10792024-08-26T20:31:20.904ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
10802024-08-26T20:31:20.904ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
10812024-08-26T20:31:20.910ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 662.782339ms iteration = 7 status = {}
10822024-08-26T20:31:30.247ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 20 reason = Timeout
10832024-08-26T20:31:30.247ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
10842024-08-26T20:31:30.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
10852024-08-26T20:31:30.248ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 20 reason = Timeout
10862024-08-26T20:31:30.248ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
10872024-08-26T20:31:30.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
10882024-08-26T20:31:30.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10892024-08-26T20:31:30.250ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
10902024-08-26T20:31:30.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
10912024-08-26T20:31:30.251ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 3.145889ms iteration = 20 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
10922024-08-26T20:31:30.252ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
10932024-08-26T20:31:30.252ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
10942024-08-26T20:31:30.252ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
10952024-08-26T20:31:30.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
10962024-08-26T20:31:30.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
10972024-08-26T20:31:30.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10982024-08-26T20:31:30.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
10992024-08-26T20:31:30.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
11002024-08-26T20:31:30.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
11012024-08-26T20:31:30.261ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
11022024-08-26T20:31:30.261ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 13.92533ms iteration = 20 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
11032024-08-26T20:31:30.874ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 212 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 287085de-5b64-484b-bb4b-fc53e2ec28c4 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
11042024-08-26T20:31:40.247ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 21 reason = Timeout
11052024-08-26T20:31:40.247ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
11062024-08-26T20:31:40.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
11072024-08-26T20:31:40.248ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 21 reason = Timeout
11082024-08-26T20:31:40.248ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
11092024-08-26T20:31:40.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
11102024-08-26T20:31:40.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11112024-08-26T20:31:40.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
11122024-08-26T20:31:40.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
11132024-08-26T20:31:40.250ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 2.544514ms iteration = 21 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
11142024-08-26T20:31:40.251ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
11152024-08-26T20:31:40.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
11162024-08-26T20:31:40.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
11172024-08-26T20:31:40.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
11182024-08-26T20:31:40.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
11192024-08-26T20:31:40.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11202024-08-26T20:31:40.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
11212024-08-26T20:31:40.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
11222024-08-26T20:31:40.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
11232024-08-26T20:31:40.263ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
11242024-08-26T20:31:40.263ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 16.122531ms iteration = 21 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
11252024-08-26T20:31:40.874ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 236 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = c7b36da1-6cfa-4166-b3e2-de977258eac9 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
11262024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 22 reason = Timeout
11272024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
11282024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 8 reason = Timeout
11292024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
11302024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 8 reason = Timeout
11312024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
11322024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 8 reason = Timeout
11332024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 8 reason = Timeout
11342024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
11352024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 8 reason = Timeout
11362024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
11372024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 8 reason = Timeout
11382024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 9 reason = Timeout
11392024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
11402024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
11412024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 8 reason = Timeout
11422024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
11432024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
11442024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
11452024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
11462024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
11472024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
11482024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
11492024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
11502024-08-26T20:31:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
11512024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 8 reason = Timeout
11522024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 8 reason = Timeout
11532024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
11542024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
11552024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 8 reason = Timeout
11562024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
11572024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 8 reason = Timeout
11582024-08-26T20:31:50.249ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task started background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:417
11592024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
11602024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 8 reason = Timeout
11612024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
11622024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 8 reason = Timeout
11632024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
11642024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
11652024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 22 reason = Timeout
11662024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
11672024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
11682024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
11692024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
11702024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
11712024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
11722024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
11732024-08-26T20:31:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
11742024-08-26T20:31:50.250ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11752024-08-26T20:31:50.250ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 2.308182ms iteration = 8 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
11762024-08-26T20:31:50.250ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1009, tv_nsec: 425701111 } })
11772024-08-26T20:31:50.250ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11782024-08-26T20:31:50.250ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1050
11792024-08-26T20:31:50.251ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11802024-08-26T20:31:50.251ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11812024-08-26T20:31:50.251ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11822024-08-26T20:31:50.252ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11832024-08-26T20:31:50.252ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
11842024-08-26T20:31:50.252ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
11852024-08-26T20:31:50.252ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
11862024-08-26T20:31:50.252ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1009, tv_nsec: 428008613 } })
11872024-08-26T20:31:50.253ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
11882024-08-26T20:31:50.253ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
11892024-08-26T20:31:50.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
11902024-08-26T20:31:50.253ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1050
11912024-08-26T20:31:50.254ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 4.842665ms iteration = 8 status = {"errors":[],"garbage_collect_requested":[]}
11922024-08-26T20:31:50.254ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
11932024-08-26T20:31:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
11942024-08-26T20:31:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
11952024-08-26T20:31:50.254ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
11962024-08-26T20:31:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
11972024-08-26T20:31:50.254ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
11982024-08-26T20:31:50.254ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11992024-08-26T20:31:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
12002024-08-26T20:31:50.254ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1009, tv_nsec: 430077582 } })
12012024-08-26T20:31:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
12022024-08-26T20:31:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
12032024-08-26T20:31:50.255ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
12042024-08-26T20:31:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
12052024-08-26T20:31:50.255ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
12062024-08-26T20:31:50.255ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
12072024-08-26T20:31:50.255ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
12082024-08-26T20:31:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
12092024-08-26T20:31:50.255ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1050 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
12102024-08-26T20:31:50.255ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 7.805553ms iteration = 8 status = {"region_replacement_started_err":0,"region_replacement_started_ok":0}
12112024-08-26T20:31:50.256ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
12122024-08-26T20:31:50.256ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
12132024-08-26T20:31:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
12142024-08-26T20:31:50.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
12152024-08-26T20:31:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
12162024-08-26T20:31:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
12172024-08-26T20:31:50.256ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
12182024-08-26T20:31:50.256ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
12192024-08-26T20:31:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
12202024-08-26T20:31:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
12212024-08-26T20:31:50.256ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
12222024-08-26T20:31:50.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
12232024-08-26T20:31:50.257ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 8.091666ms iteration = 22 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
12242024-08-26T20:31:50.257ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12252024-08-26T20:31:50.258ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
12262024-08-26T20:31:50.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
12272024-08-26T20:31:50.258ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
12282024-08-26T20:31:50.259ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
12292024-08-26T20:31:50.259ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
12302024-08-26T20:31:50.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
12312024-08-26T20:31:50.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
12322024-08-26T20:31:50.259ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12332024-08-26T20:31:50.261ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:454 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
12342024-08-26T20:31:50.261ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 11.947732ms iteration = 8 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
12352024-08-26T20:31:50.263ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
12362024-08-26T20:31:50.263ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
12372024-08-26T20:31:50.263ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
12382024-08-26T20:31:50.280ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
12392024-08-26T20:31:50.280ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 32.029169ms iteration = 22 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
12402024-08-26T20:31:50.282ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
12412024-08-26T20:31:50.282ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
12422024-08-26T20:31:50.283ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
12432024-08-26T20:31:50.284ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
12442024-08-26T20:31:50.285ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12452024-08-26T20:31:50.287ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: c9d3ee27-fe3f-422b-93ce-695a8356d9d2, lookup_type: ById(c9d3ee27-fe3f-422b-93ce-695a8356d9d2) } result = Ok(())
12462024-08-26T20:31:50.287ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
12472024-08-26T20:31:50.287ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
12482024-08-26T20:31:50.287ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
12492024-08-26T20:31:50.287ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
12502024-08-26T20:31:50.288ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 40.457329ms iteration = 8 status = 0
12512024-08-26T20:31:50.289ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
12522024-08-26T20:31:50.289ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
12532024-08-26T20:31:50.290ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task done background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:439
12542024-08-26T20:31:50.290ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 40.899592ms iteration = 8 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
12552024-08-26T20:31:50.297ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
12562024-08-26T20:31:50.297ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:220 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
12572024-08-26T20:31:50.297ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 48.234781ms iteration = 8 status = {"destroyed_active_vmms":0,"error":null,"saga_start_failures":0,"sagas_completed":0,"sagas_failed":0,"sagas_started":0,"terminated_active_migrations":0}
12582024-08-26T20:31:50.298ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 50.196969ms iteration = 9 status = {"physical_disks_added":0}
12592024-08-26T20:31:50.299ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
12602024-08-26T20:31:50.299ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
12612024-08-26T20:31:50.300ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 50.928947ms iteration = 8 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
12622024-08-26T20:31:50.306ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
12632024-08-26T20:31:50.306ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
12642024-08-26T20:31:50.308ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12652024-08-26T20:31:50.309ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
12662024-08-26T20:31:50.309ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
12672024-08-26T20:31:50.309ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
12682024-08-26T20:31:50.310ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12692024-08-26T20:31:50.311ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
12702024-08-26T20:31:50.311ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
12712024-08-26T20:31:50.312ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
12722024-08-26T20:31:50.360ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
12732024-08-26T20:31:50.360ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
12742024-08-26T20:31:50.361ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "d293deb0-12bb-4bdf-b7ea-ec790494107a", "content-length": "26", "date": "Mon, 26 Aug 2024 20:31:50 GMT"} })
12752024-08-26T20:31:50.361ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
12762024-08-26T20:31:50.363ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
12772024-08-26T20:31:50.391ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
12782024-08-26T20:31:50.391ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
12792024-08-26T20:31:50.392ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "7bcffcb1-66ee-457a-adaf-769029ebc4f2", "content-length": "26", "date": "Mon, 26 Aug 2024 20:31:50 GMT"} })
12802024-08-26T20:31:50.392ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
12812024-08-26T20:31:50.392ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1058 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 zone_address = fd00:1122:3344:101::2
12822024-08-26T20:31:50.392ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
12832024-08-26T20:31:50.393ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "fb26fdb8-c6b7-40a9-8106-541dcd2597e2", "content-length": "26", "date": "Mon, 26 Aug 2024 20:31:50 GMT"} })
12842024-08-26T20:31:50.393ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1063 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
12852024-08-26T20:31:50.394ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
12862024-08-26T20:31:50.395ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
12872024-08-26T20:31:50.395ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "d622671c-e355-4e0d-be1e-a9df72481003", "content-length": "372", "date": "Mon, 26 Aug 2024 20:31:49 GMT"} })
12882024-08-26T20:31:50.395ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
12892024-08-26T20:31:50.395ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
12902024-08-26T20:31:50.396ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
12912024-08-26T20:31:50.397ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
12922024-08-26T20:31:50.397ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
12932024-08-26T20:31:50.399ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
12942024-08-26T20:31:50.399ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
12952024-08-26T20:31:50.401ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
12962024-08-26T20:31:50.402ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 204, headers: {"x-request-id": "fb5a7225-8dda-4ced-af36-b8c872ba55e3", "date": "Mon, 26 Aug 2024 20:31:49 GMT"} })
12972024-08-26T20:31:50.404ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 155.199271ms iteration = 8 status = {}
12982024-08-26T20:31:50.446ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
12992024-08-26T20:31:50.446ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/v2p", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "d2b1a259-0bda-4dca-8f6a-eb16a34c8af5", "content-length": "691", "date": "Mon, 26 Aug 2024 20:31:50 GMT"} })
13002024-08-26T20:31:50.446ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = sock
13012024-08-26T20:31:50.446ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = sock
13022024-08-26T20:31:50.446ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = sock
13032024-08-26T20:31:50.448ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 199.812499ms iteration = 8 status = {}
13042024-08-26T20:31:50.518ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 270.046856ms iteration = 8 status = {}
13052024-08-26T20:31:50.565ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
13062024-08-26T20:31:50.565ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "e1f8ea70-506d-4781-b3c5-d6a064ae0a6a", "content-length": "1", "date": "Mon, 26 Aug 2024 20:31:50 GMT"} })
13072024-08-26T20:31:50.566ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
13082024-08-26T20:31:50.567ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
13092024-08-26T20:31:50.568ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 320.076173ms iteration = 8 status = null
13102024-08-26T20:31:50.637ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
13112024-08-26T20:31:50.637ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
13122024-08-26T20:31:50.639ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:147 port = SwitchPort { id: 77948b01-64a9-403e-be7b-4ef5e426dda4, rack_id: 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(735476d6-05ca-4fbd-a182-ab082f700355) } rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
13132024-08-26T20:31:50.639ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
13142024-08-26T20:31:50.639ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
13152024-08-26T20:31:50.647ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
13162024-08-26T20:31:50.647ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/port/qsfp0/settings", query: Some("tag=omicron"), fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "ecc3e053-7821-407d-ae0f-0e296b7f186b", "content-length": "121", "date": "Mon, 26 Aug 2024 20:31:50 GMT"} })
13172024-08-26T20:31:50.647ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1714 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.2.174], params: LinkCreate { autoneg: false, fec: None, kr: false, lane: Some(LinkId(0)), speed: Speed40G } }} } switch_port_id = "qsfp0"
13182024-08-26T20:31:50.647ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1741 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = 735476d6-05ca-4fbd-a182-ab082f700355
13192024-08-26T20:31:50.647ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:379 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
13202024-08-26T20:31:50.647ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
13212024-08-26T20:31:50.647ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:392 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
13222024-08-26T20:31:50.647ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:398 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
13232024-08-26T20:31:50.647ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:418 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
13242024-08-26T20:31:50.647ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
13252024-08-26T20:31:50.648ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
13262024-08-26T20:31:50.649ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
13272024-08-26T20:31:50.649ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv4", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "88c8f234-305b-408b-ab51-1cb3198e5709", "content-length": "2", "date": "Mon, 26 Aug 2024 20:31:50 GMT"} })
13282024-08-26T20:31:50.649ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
13292024-08-26T20:31:50.650ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv6", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "7ef81035-a7fd-46a7-ac1f-7a730349082c", "content-length": "52", "date": "Mon, 26 Aug 2024 20:31:50 GMT"} })
13302024-08-26T20:31:50.650ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0" }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:467 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
13312024-08-26T20:31:50.650ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
13322024-08-26T20:31:50.741ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/switch-ports", query: None, fragment: None }, status: 204, headers: {"x-request-id": "6152bed6-1dcb-4eea-a66a-f26528394abb", "date": "Mon, 26 Aug 2024 20:31:50 GMT"} })
13332024-08-26T20:31:50.741ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:770 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch = Switch0
13342024-08-26T20:31:50.741ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
13352024-08-26T20:31:50.742ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/network-bootstore-config", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "f583351b-4cea-432b-90c6-518b24e26630", "content-length": "667", "date": "Mon, 26 Aug 2024 20:31:50 GMT"} })
13362024-08-26T20:31:50.742ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
13372024-08-26T20:31:50.742ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
13382024-08-26T20:31:50.746ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
13392024-08-26T20:31:50.746ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
13402024-08-26T20:31:50.749ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
13412024-08-26T20:31:50.749ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
13422024-08-26T20:31:50.750ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
13432024-08-26T20:31:50.750ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
13442024-08-26T20:31:50.756ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 508.502126ms iteration = 8 status = {}
13452024-08-26T20:31:50.851ZDEBGnexus (producer-registration-task): registering / renewing oximeter producer lease with Nexus
13462024-08-26T20:31:50.851ZDEBGnexus (producer-registration-task): using nexus address for registration addr = [fd00:1122:3344:101::c]:12221
13472024-08-26T20:31:50.881ZDEBGnexus (producer-registration-task): client request body = Some(Body) method = POST uri = http://[fd00:1122:3344:101::c]:12221/metrics/producers
13482024-08-26T20:31:50.881ZINFOnexus (dropshot_internal): accepted connection file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775 local_addr = [fd00:1122:3344:101::c]:12221 remote_addr = [fd00:1122:3344:101::c]:51890
13492024-08-26T20:31:50.882ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:51890 req_id = e20e5c7b-eed3-4d88-9259-54eda9efc540 roles = RoleSet { roles: {} } uri = /metrics/producers
13502024-08-26T20:31:50.882ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 772 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 6ba4faf4-b403-4937-8f88-a28b0e222262 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
13512024-08-26T20:31:50.882ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:51890 req_id = e20e5c7b-eed3-4d88-9259-54eda9efc540 resource = Database result = Ok(()) uri = /metrics/producers
13522024-08-26T20:31:50.915ZINFOnexus: registered oximeter collector client file = nexus/src/app/oximeter.rs:338 id = 66888b78-0269-44c1-8d42-ab8ec11561af
13532024-08-26T20:31:50.915ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:51890 req_id = e20e5c7b-eed3-4d88-9259-54eda9efc540 roles = RoleSet { roles: {} } uri = /metrics/producers
13542024-08-26T20:31:50.915ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:51890 req_id = e20e5c7b-eed3-4d88-9259-54eda9efc540 resource = Database result = Ok(()) uri = /metrics/producers
13552024-08-26T20:31:50.979ZDEBGnexus: client request body = Some(Body) method = POST oximeter-collector = 66888b78-0269-44c1-8d42-ab8ec11561af uri = http://[fd00:1122:3344:101::d]:12223/producers
13562024-08-26T20:31:50.979ZDEBGnexus: client response oximeter-collector = 66888b78-0269-44c1-8d42-ab8ec11561af result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::d)), port: Some(12223), path: "/producers", query: None, fragment: None }, status: 204, headers: {"x-request-id": "0dbb008f-a371-4ba1-947a-d79132b33844", "date": "Mon, 26 Aug 2024 20:31:50 GMT"} })
13572024-08-26T20:31:50.979ZINFOnexus: assigned collector to new producer collector_id = 66888b78-0269-44c1-8d42-ab8ec11561af file = nexus/src/app/oximeter.rs:125 producer_id = ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
13582024-08-26T20:31:50.980ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 98985 local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:51890 req_id = e20e5c7b-eed3-4d88-9259-54eda9efc540 response_code = 201 uri = /metrics/producers
13592024-08-26T20:31:50.981ZDEBGnexus (producer-registration-task): client response result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, status: 201, headers: {"content-type": "application/json", "x-request-id": "e20e5c7b-eed3-4d88-9259-54eda9efc540", "content-length": "41", "date": "Mon, 26 Aug 2024 20:31:50 GMT"} })
13602024-08-26T20:31:50.982ZDEBGnexus (producer-registration-task): registered with nexus successfully
13612024-08-26T20:31:50.982ZDEBGnexus (producer-registration-task): pausing until time to renew lease lease_duration = 600s wait_period = 150s
13622024-08-26T20:32:00.247ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 23 reason = Timeout
13632024-08-26T20:32:00.247ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
13642024-08-26T20:32:00.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
13652024-08-26T20:32:00.248ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 23 reason = Timeout
13662024-08-26T20:32:00.248ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
13672024-08-26T20:32:00.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
13682024-08-26T20:32:00.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
13692024-08-26T20:32:00.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
13702024-08-26T20:32:00.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
13712024-08-26T20:32:00.250ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 2.443723ms iteration = 23 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
13722024-08-26T20:32:00.251ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
13732024-08-26T20:32:00.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
13742024-08-26T20:32:00.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
13752024-08-26T20:32:00.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
13762024-08-26T20:32:00.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
13772024-08-26T20:32:00.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
13782024-08-26T20:32:00.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
13792024-08-26T20:32:00.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
13802024-08-26T20:32:00.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
13812024-08-26T20:32:00.262ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
13822024-08-26T20:32:00.262ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 14.247943ms iteration = 23 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
13832024-08-26T20:32:00.981ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 208 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = bb228f4c-a52f-4ca5-90aa-2878f332294a response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
13842024-08-26T20:32:10.247ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 24 reason = Timeout
13852024-08-26T20:32:10.247ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
13862024-08-26T20:32:10.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
13872024-08-26T20:32:10.248ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 24 reason = Timeout
13882024-08-26T20:32:10.248ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
13892024-08-26T20:32:10.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
13902024-08-26T20:32:10.250ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
13912024-08-26T20:32:10.250ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
13922024-08-26T20:32:10.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
13932024-08-26T20:32:10.251ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 2.387282ms iteration = 24 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
13942024-08-26T20:32:10.252ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
13952024-08-26T20:32:10.252ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
13962024-08-26T20:32:10.252ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
13972024-08-26T20:32:10.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
13982024-08-26T20:32:10.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
13992024-08-26T20:32:10.255ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
14002024-08-26T20:32:10.257ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
14012024-08-26T20:32:10.257ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
14022024-08-26T20:32:10.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
14032024-08-26T20:32:10.262ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
14042024-08-26T20:32:10.262ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 14.87514ms iteration = 24 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
14052024-08-26T20:32:10.981ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 212 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 5abbe4fc-4b97-4240-bfb5-014fae920a20 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
14062024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 5 reason = Timeout
14072024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 5 reason = Timeout
14082024-08-26T20:32:20.248ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-08-26 20:22:20.248628802 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
14092024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
14102024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 9 reason = Timeout
14112024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 7 reason = Timeout
14122024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
14132024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 5 reason = Timeout
14142024-08-26T20:32:20.248ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
14152024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 9 reason = Timeout
14162024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
14172024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
14182024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 7 reason = Timeout
14192024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 5 reason = Timeout
14202024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 9 reason = Timeout
14212024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
14222024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 9 reason = Timeout
14232024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 9 reason = Timeout
14242024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
14252024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 9 reason = Timeout
14262024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
14272024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
14282024-08-26T20:32:20.248ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 25 reason = Timeout
14292024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
14302024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 6 reason = Timeout
14312024-08-26T20:32:20.249ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:80 reason = blueprint disabled target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460
14322024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 9 reason = Timeout
14332024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 16.89\u{b5}s iteration = 6 status = {"error":"blueprint disabled","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460"}
14342024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
14352024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 5 reason = Timeout
14362024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
14372024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 5 reason = Timeout
14382024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
14392024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 9 reason = Timeout
14402024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
14412024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 5 reason = Timeout
14422024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
14432024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
14442024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 10 reason = Timeout
14452024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 25 reason = Timeout
14462024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
14472024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
14482024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 9 reason = Timeout
14492024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
14502024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
14512024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 9 reason = Timeout
14522024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
14532024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 5 reason = Timeout
14542024-08-26T20:32:20.249ZINFOnexus (BackgroundTasks): Abandoned VMM reaper running background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:76
14552024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
14562024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 9 reason = Timeout
14572024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
14582024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 9 reason = Timeout
14592024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
14602024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 9 reason = Timeout
14612024-08-26T20:32:20.249ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task started background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:417
14622024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
14632024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 5 reason = Timeout
14642024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
14652024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
14662024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
14672024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
14682024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
14692024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
14702024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
14712024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
14722024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
14732024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
14742024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
14752024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
14762024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
14772024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
14782024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
14792024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
14802024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
14812024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
14822024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
14832024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
14842024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
14852024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
14862024-08-26T20:32:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
14872024-08-26T20:32:20.250ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("be00167b-db1f-4cc1-86f1-a2af7f1c31f8.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("d2b5c33b-9149-4357-86b1-60e922261ec4.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("d2b5c33b-9149-4357-86b1-60e922261ec4.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 1039, tv_nsec: 425702941 } })
14882024-08-26T20:32:20.250ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("316f64f2-8fa5-43f3-8d5d-92212e3046c4.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("35cb7d67-b9f1-4dcf-bac3-28244fb25d88.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("e37633b8-b58b-40c2-b3c2-6873358565b7.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("e37633b8-b58b-40c2-b3c2-6873358565b7.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:3::1))) }], valid_until: Instant { tv_sec: 1039, tv_nsec: 425853993 } })
14892024-08-26T20:32:20.251ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
14902024-08-26T20:32:20.252ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
14912024-08-26T20:32:20.252ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.628264ms iteration = 9 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
14922024-08-26T20:32:20.252ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 3.437712ms iteration = 5 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
14932024-08-26T20:32:20.252ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
14942024-08-26T20:32:20.252ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
14952024-08-26T20:32:20.253ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
14962024-08-26T20:32:20.253ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
14972024-08-26T20:32:20.253ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
14982024-08-26T20:32:20.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
14992024-08-26T20:32:20.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
15002024-08-26T20:32:20.253ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
15012024-08-26T20:32:20.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15022024-08-26T20:32:20.254ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-08-26 20:22:20.248628802 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
15032024-08-26T20:32:20.254ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 5.500382ms iteration = 5 status = {"expiration":"2024-08-26T20:22:20.248628802Z","pruned":[]}
15042024-08-26T20:32:20.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
15052024-08-26T20:32:20.254ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } dns_group = internal
15062024-08-26T20:32:20.254ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 5.781684ms iteration = 5 status = {"addresses":["[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353","[fd00:1122:3344:3::1]:5353"]}
15072024-08-26T20:32:20.254ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1039, tv_nsec: 429410286 } })
15082024-08-26T20:32:20.254ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1039, tv_nsec: 429440746 } })
15092024-08-26T20:32:20.254ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15102024-08-26T20:32:20.254ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
15112024-08-26T20:32:20.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
15122024-08-26T20:32:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
15132024-08-26T20:32:20.255ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
15142024-08-26T20:32:20.255ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1050
15152024-08-26T20:32:20.255ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 6.369149ms iteration = 9 status = {"errors":[],"garbage_collect_requested":[]}
15162024-08-26T20:32:20.255ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1050
15172024-08-26T20:32:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
15182024-08-26T20:32:20.255ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
15192024-08-26T20:32:20.255ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15202024-08-26T20:32:20.255ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15212024-08-26T20:32:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
15222024-08-26T20:32:20.256ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 7.187457ms iteration = 9 status = {"region_replacement_started_err":0,"region_replacement_started_ok":0}
15232024-08-26T20:32:20.256ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15242024-08-26T20:32:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
15252024-08-26T20:32:20.256ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
15262024-08-26T20:32:20.256ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
15272024-08-26T20:32:20.257ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 8.212986ms iteration = 9 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
15282024-08-26T20:32:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
15292024-08-26T20:32:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
15302024-08-26T20:32:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
15312024-08-26T20:32:20.257ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
15322024-08-26T20:32:20.257ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
15332024-08-26T20:32:20.257ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
15342024-08-26T20:32:20.257ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
15352024-08-26T20:32:20.257ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0
15362024-08-26T20:32:20.257ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:189 found = 0 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0
15372024-08-26T20:32:20.257ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 8.57496ms iteration = 5 status = {"error":null,"error_count":0,"found":0,"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0}
15382024-08-26T20:32:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
15392024-08-26T20:32:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
15402024-08-26T20:32:20.258ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
15412024-08-26T20:32:20.258ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
15422024-08-26T20:32:20.258ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
15432024-08-26T20:32:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
15442024-08-26T20:32:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
15452024-08-26T20:32:20.258ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
15462024-08-26T20:32:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
15472024-08-26T20:32:20.258ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15482024-08-26T20:32:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
15492024-08-26T20:32:20.259ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
15502024-08-26T20:32:20.259ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
15512024-08-26T20:32:20.259ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
15522024-08-26T20:32:20.259ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
15532024-08-26T20:32:20.259ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
15542024-08-26T20:32:20.259ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
15552024-08-26T20:32:20.259ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15562024-08-26T20:32:20.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
15572024-08-26T20:32:20.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
15582024-08-26T20:32:20.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
15592024-08-26T20:32:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
15602024-08-26T20:32:20.260ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
15612024-08-26T20:32:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
15622024-08-26T20:32:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
15632024-08-26T20:32:20.260ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15642024-08-26T20:32:20.260ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
15652024-08-26T20:32:20.260ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
15662024-08-26T20:32:20.260ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
15672024-08-26T20:32:20.260ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
15682024-08-26T20:32:20.260ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15692024-08-26T20:32:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
15702024-08-26T20:32:20.260ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
15712024-08-26T20:32:20.261ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
15722024-08-26T20:32:20.261ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
15732024-08-26T20:32:20.262ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
15742024-08-26T20:32:20.262ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1039, tv_nsec: 437245639 } })
15752024-08-26T20:32:20.262ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
15762024-08-26T20:32:20.262ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
15772024-08-26T20:32:20.262ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15782024-08-26T20:32:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
15792024-08-26T20:32:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
15802024-08-26T20:32:20.262ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
15812024-08-26T20:32:20.263ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
15822024-08-26T20:32:20.263ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1050 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
15832024-08-26T20:32:20.263ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
15842024-08-26T20:32:20.263ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
15852024-08-26T20:32:20.263ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
15862024-08-26T20:32:20.263ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
15872024-08-26T20:32:20.263ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
15882024-08-26T20:32:20.264ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15892024-08-26T20:32:20.264ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
15902024-08-26T20:32:20.264ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
15912024-08-26T20:32:20.264ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task done background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:439
15922024-08-26T20:32:20.264ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 15.193802ms iteration = 9 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
15932024-08-26T20:32:20.264ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
15942024-08-26T20:32:20.264ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15952024-08-26T20:32:20.265ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
15962024-08-26T20:32:20.265ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
15972024-08-26T20:32:20.265ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
15982024-08-26T20:32:20.265ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
15992024-08-26T20:32:20.265ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
16002024-08-26T20:32:20.265ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
16012024-08-26T20:32:20.266ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
16022024-08-26T20:32:20.266ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = d840a78c-2f0d-4cdf-9962-31a3dae384a2 dns_zone_name = oxide.test found_so_far = 0 version = 2
16032024-08-26T20:32:20.266ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
16042024-08-26T20:32:20.266ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
16052024-08-26T20:32:20.266ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:454 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
16062024-08-26T20:32:20.266ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 17.325992ms iteration = 9 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
16072024-08-26T20:32:20.266ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
16082024-08-26T20:32:20.266ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = b82a4171-23b2-444b-9597-09085a7a2fb8 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
16092024-08-26T20:32:20.266ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
16102024-08-26T20:32:20.267ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
16112024-08-26T20:32:20.267ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16122024-08-26T20:32:20.267ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
16132024-08-26T20:32:20.267ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
16142024-08-26T20:32:20.267ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
16152024-08-26T20:32:20.268ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16162024-08-26T20:32:20.269ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
16172024-08-26T20:32:20.269ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
16182024-08-26T20:32:20.269ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
16192024-08-26T20:32:20.269ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
16202024-08-26T20:32:20.269ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
16212024-08-26T20:32:20.270ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
16222024-08-26T20:32:20.270ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
16232024-08-26T20:32:20.270ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
16242024-08-26T20:32:20.270ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 21.42066ms iteration = 25 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
16252024-08-26T20:32:20.270ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16262024-08-26T20:32:20.270ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
16272024-08-26T20:32:20.270ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 21.667963ms iteration = 10 status = {"physical_disks_added":0}
16282024-08-26T20:32:20.270ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
16292024-08-26T20:32:20.270ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
16302024-08-26T20:32:20.270ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
16312024-08-26T20:32:20.270ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 21.734723ms iteration = 5 status = {"errors":[],"found_port_ok":[]}
16322024-08-26T20:32:20.271ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
16332024-08-26T20:32:20.271ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16342024-08-26T20:32:20.271ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 22.42705ms iteration = 25 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
16352024-08-26T20:32:20.271ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
16362024-08-26T20:32:20.272ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = d840a78c-2f0d-4cdf-9962-31a3dae384a2 dns_zone_name = oxide.test found_so_far = 1 version = 2
16372024-08-26T20:32:20.272ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
16382024-08-26T20:32:20.272ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-08-26 20:29:01.487349 UTC dns_group = external generation = 2
16392024-08-26T20:32:20.272ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 23.142926ms iteration = 5 status = {"generation":2}
16402024-08-26T20:32:20.272ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16412024-08-26T20:32:20.272ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
16422024-08-26T20:32:20.272ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
16432024-08-26T20:32:20.272ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
16442024-08-26T20:32:20.272ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
16452024-08-26T20:32:20.272ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = b82a4171-23b2-444b-9597-09085a7a2fb8 dns_zone_name = control-plane.oxide.internal found_so_far = 50 version = 1
16462024-08-26T20:32:20.272ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
16472024-08-26T20:32:20.272ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-08-26 20:29:01.089496 UTC dns_group = internal generation = 1
16482024-08-26T20:32:20.272ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 23.921954ms iteration = 5 status = {"generation":1}
16492024-08-26T20:32:20.273ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
16502024-08-26T20:32:20.273ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
16512024-08-26T20:32:20.273ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16522024-08-26T20:32:20.273ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
16532024-08-26T20:32:20.273ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
16542024-08-26T20:32:20.273ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
16552024-08-26T20:32:20.273ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
16562024-08-26T20:32:20.274ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16572024-08-26T20:32:20.275ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: c9d3ee27-fe3f-422b-93ce-695a8356d9d2, lookup_type: ById(c9d3ee27-fe3f-422b-93ce-695a8356d9d2) } result = Ok(())
16582024-08-26T20:32:20.275ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
16592024-08-26T20:32:20.275ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
16602024-08-26T20:32:20.276ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
16612024-08-26T20:32:20.276ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
16622024-08-26T20:32:20.276ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
16632024-08-26T20:32:20.277ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 28.477516ms iteration = 9 status = 0
16642024-08-26T20:32:20.281ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
16652024-08-26T20:32:20.281ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
16662024-08-26T20:32:20.281ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 32.651496ms iteration = 5 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
16672024-08-26T20:32:20.285ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
16682024-08-26T20:32:20.285ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:220 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
16692024-08-26T20:32:20.285ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 36.208378ms iteration = 9 status = {"destroyed_active_vmms":0,"error":null,"saga_start_failures":0,"sagas_completed":0,"sagas_failed":0,"sagas_started":0,"terminated_active_migrations":0}
16702024-08-26T20:32:20.306ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-08-26T20:28:18.089116Z, time_modified: 2024-08-26T20:28:18.089116Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 80a72585-f38a-4ccb-b69a-caafab415ca2, db_silo: Silo { identity: SiloIdentity { id: 80a72585-f38a-4ccb-b69a-caafab415ca2, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-08-26T20:29:01.450089Z, time_modified: 2024-08-26T20:29:01.450089Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 80a72585-f38a-4ccb-b69a-caafab415ca2, db_silo: Silo { identity: SiloIdentity { id: 80a72585-f38a-4ccb-b69a-caafab415ca2, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-08-26T20:29:01.450089Z, time_modified: 2024-08-26T20:29:01.450089Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923" }] }) }
16712024-08-26T20:32:20.306ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 58.108443ms iteration = 5 status = {"by_dns_name":{"default-silo.sys.oxide.test":{"silo_id":"001de000-5110-4000-8000-000000000000","tls_certs":[]},"recovery.sys.oxide.test":{"silo_id":"80a72585-f38a-4ccb-b69a-caafab415ca2","tls_certs":["81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923"]}},"default_endpoint":{"silo_id":"80a72585-f38a-4ccb-b69a-caafab415ca2","tls_certs":["81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923"]},"warnings":["silo 001de000-5110-4000-8000-000000000000 with DNS name \\"default-silo.sys.oxide.test\\" has no usable certificates"]}
16722024-08-26T20:32:20.450ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
16732024-08-26T20:32:20.450ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
16742024-08-26T20:32:20.450ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "4a547022-19db-4102-bb49-84651ade13a3", "content-length": "26", "date": "Mon, 26 Aug 2024 20:32:20 GMT"} })
16752024-08-26T20:32:20.450ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
16762024-08-26T20:32:20.452ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
16772024-08-26T20:32:20.614ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
16782024-08-26T20:32:20.616ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
16792024-08-26T20:32:20.619ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1058 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 zone_address = fd00:1122:3344:101::2
16802024-08-26T20:32:20.619ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
16812024-08-26T20:32:20.620ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
16822024-08-26T20:32:20.620ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
16832024-08-26T20:32:20.620ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "952ab875-485b-4148-ab7f-8d7b0c1a51ef", "content-length": "26", "date": "Mon, 26 Aug 2024 20:32:20 GMT"} })
16842024-08-26T20:32:20.620ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1063 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
16852024-08-26T20:32:20.620ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "df1c9c4b-2101-467b-9967-1e40c6b91d37", "content-length": "26", "date": "Mon, 26 Aug 2024 20:32:20 GMT"} })
16862024-08-26T20:32:20.620ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
16872024-08-26T20:32:20.625ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
16882024-08-26T20:32:20.625ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
16892024-08-26T20:32:20.639ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
16902024-08-26T20:32:20.639ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/v2p", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "c4c49b7e-d212-4fd6-adc2-2638ee037fcf", "content-length": "691", "date": "Mon, 26 Aug 2024 20:32:19 GMT"} })
16912024-08-26T20:32:20.639ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = sock
16922024-08-26T20:32:20.639ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = sock
16932024-08-26T20:32:20.639ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = sock
16942024-08-26T20:32:20.641ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 392.705913ms iteration = 9 status = {}
16952024-08-26T20:32:20.647ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
16962024-08-26T20:32:20.647ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "144b840a-a727-43e3-b677-962ef33480ec", "content-length": "372", "date": "Mon, 26 Aug 2024 20:32:19 GMT"} })
16972024-08-26T20:32:20.647ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
16982024-08-26T20:32:20.647ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
16992024-08-26T20:32:20.649ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
17002024-08-26T20:32:20.649ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
17012024-08-26T20:32:20.651ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
17022024-08-26T20:32:20.651ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
17032024-08-26T20:32:20.653ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
17042024-08-26T20:32:20.653ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 204, headers: {"x-request-id": "bcf95c3f-d7ec-436f-bd4b-614d5453278e", "date": "Mon, 26 Aug 2024 20:32:19 GMT"} })
17052024-08-26T20:32:20.654ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 404.900657ms iteration = 9 status = {}
17062024-08-26T20:32:20.658ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
17072024-08-26T20:32:20.658ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "bd108e00-8118-4854-9b9c-c63e38f61651", "content-length": "1", "date": "Mon, 26 Aug 2024 20:32:20 GMT"} })
17082024-08-26T20:32:20.660ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
17092024-08-26T20:32:20.660ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
17102024-08-26T20:32:20.661ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 412.612639ms iteration = 9 status = null
17112024-08-26T20:32:20.750ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 501.573941ms iteration = 9 status = {}
17122024-08-26T20:32:20.827ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
17132024-08-26T20:32:20.871ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
17142024-08-26T20:32:20.872ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::8)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "bca8b027-2ddf-45c8-8d52-6323ea7b5d8b", "date": "Mon, 26 Aug 2024 20:32:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
17152024-08-26T20:32:20.872ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
17162024-08-26T20:32:20.874ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::9)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "de2cda17-0832-4b5e-9108-f42af1636772", "date": "Mon, 26 Aug 2024 20:32:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
17172024-08-26T20:32:20.874ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
17182024-08-26T20:32:20.876ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
17192024-08-26T20:32:20.876ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 627.487489ms iteration = 7 status = {"generation":2,"server_results":{"[fd00:1122:3344:101::8]:5353":{"Ok":null},"[fd00:1122:3344:101::9]:5353":{"Ok":null}}}
17202024-08-26T20:32:20.898ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
17212024-08-26T20:32:20.898ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
17222024-08-26T20:32:20.899ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
17232024-08-26T20:32:20.899ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:3::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "f79d7095-7648-489b-ae12-f9252d55686f", "date": "Mon, 26 Aug 2024 20:32:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
17242024-08-26T20:32:20.899ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
17252024-08-26T20:32:20.900ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:147 port = SwitchPort { id: 77948b01-64a9-403e-be7b-4ef5e426dda4, rack_id: 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(735476d6-05ca-4fbd-a182-ab082f700355) } rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
17262024-08-26T20:32:20.900ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
17272024-08-26T20:32:20.900ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:2::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "3f1a03d1-54de-4537-b34e-4d9089b57fed", "date": "Mon, 26 Aug 2024 20:32:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
17282024-08-26T20:32:20.900ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
17292024-08-26T20:32:20.901ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
17302024-08-26T20:32:20.902ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:1::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "31f41b4d-7c1b-470c-8646-eb747f671f1d", "date": "Mon, 26 Aug 2024 20:32:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
17312024-08-26T20:32:20.902ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
17322024-08-26T20:32:20.904ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
17332024-08-26T20:32:20.904ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 656.078845ms iteration = 7 status = {"generation":1,"server_results":{"[fd00:1122:3344:1::1]:5353":{"Ok":null},"[fd00:1122:3344:2::1]:5353":{"Ok":null},"[fd00:1122:3344:3::1]:5353":{"Ok":null}}}
17342024-08-26T20:32:20.909ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
17352024-08-26T20:32:20.909ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/port/qsfp0/settings", query: Some("tag=omicron"), fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "3556c41b-5e8e-44ba-8dd3-7d7a94402cc7", "content-length": "121", "date": "Mon, 26 Aug 2024 20:32:20 GMT"} })
17362024-08-26T20:32:20.909ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1714 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.2.174], params: LinkCreate { autoneg: false, fec: None, kr: false, lane: Some(LinkId(0)), speed: Speed40G } }} } switch_port_id = "qsfp0"
17372024-08-26T20:32:20.909ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1741 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = 735476d6-05ca-4fbd-a182-ab082f700355
17382024-08-26T20:32:20.909ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:379 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
17392024-08-26T20:32:20.909ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
17402024-08-26T20:32:20.909ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:392 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
17412024-08-26T20:32:20.909ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:398 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
17422024-08-26T20:32:20.909ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:418 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
17432024-08-26T20:32:20.909ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
17442024-08-26T20:32:20.909ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
17452024-08-26T20:32:20.911ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
17462024-08-26T20:32:20.912ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv4", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "2a604abb-8277-40a7-bae2-aee5679a88e9", "content-length": "2", "date": "Mon, 26 Aug 2024 20:32:20 GMT"} })
17472024-08-26T20:32:20.912ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
17482024-08-26T20:32:20.912ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv6", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "902dd653-893e-43da-b149-a0deaf5f29d9", "content-length": "52", "date": "Mon, 26 Aug 2024 20:32:20 GMT"} })
17492024-08-26T20:32:20.912ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0" }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:467 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
17502024-08-26T20:32:20.912ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
17512024-08-26T20:32:20.981ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 190 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = c6b4fe7c-05e8-4f98-93d6-eaa97efc278f response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
17522024-08-26T20:32:20.999ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/switch-ports", query: None, fragment: None }, status: 204, headers: {"x-request-id": "7edaecfd-85c4-4732-9581-73dbcc3241ea", "date": "Mon, 26 Aug 2024 20:32:20 GMT"} })
17532024-08-26T20:32:20.999ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:770 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch = Switch0
17542024-08-26T20:32:20.999ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
17552024-08-26T20:32:21.000ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/network-bootstore-config", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "3af3b732-675c-4d77-b3e5-0701b0e4d949", "content-length": "667", "date": "Mon, 26 Aug 2024 20:32:20 GMT"} })
17562024-08-26T20:32:21.000ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
17572024-08-26T20:32:21.000ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
17582024-08-26T20:32:21.003ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
17592024-08-26T20:32:21.004ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
17602024-08-26T20:32:21.006ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
17612024-08-26T20:32:21.006ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
17622024-08-26T20:32:21.008ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
17632024-08-26T20:32:21.008ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
17642024-08-26T20:32:21.014ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 765.728672ms iteration = 9 status = {}
17652024-08-26T20:32:30.247ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 26 reason = Timeout
17662024-08-26T20:32:30.247ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
17672024-08-26T20:32:30.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
17682024-08-26T20:32:30.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
17692024-08-26T20:32:30.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 26 reason = Timeout
17702024-08-26T20:32:30.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
17712024-08-26T20:32:30.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
17722024-08-26T20:32:30.250ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
17732024-08-26T20:32:30.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
17742024-08-26T20:32:30.251ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
17752024-08-26T20:32:30.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
17762024-08-26T20:32:30.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
17772024-08-26T20:32:30.251ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 2.393722ms iteration = 26 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
17782024-08-26T20:32:30.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
17792024-08-26T20:32:30.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
17802024-08-26T20:32:30.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
17812024-08-26T20:32:30.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
17822024-08-26T20:32:30.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
17832024-08-26T20:32:30.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
17842024-08-26T20:32:30.262ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
17852024-08-26T20:32:30.262ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 14.864959ms iteration = 26 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
17862024-08-26T20:32:30.981ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 208 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 578e6ef7-1783-44bd-b2f1-d52ad37b3be9 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
17872024-08-26T20:32:40.248ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 27 reason = Timeout
17882024-08-26T20:32:40.248ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
17892024-08-26T20:32:40.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
17902024-08-26T20:32:40.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 27 reason = Timeout
17912024-08-26T20:32:40.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
17922024-08-26T20:32:40.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
17932024-08-26T20:32:40.250ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
17942024-08-26T20:32:40.250ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
17952024-08-26T20:32:40.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
17962024-08-26T20:32:40.251ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 2.531723ms iteration = 27 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
17972024-08-26T20:32:40.253ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
17982024-08-26T20:32:40.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
17992024-08-26T20:32:40.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
18002024-08-26T20:32:40.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
18012024-08-26T20:32:40.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
18022024-08-26T20:32:40.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
18032024-08-26T20:32:40.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
18042024-08-26T20:32:40.258ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
18052024-08-26T20:32:40.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
18062024-08-26T20:32:40.264ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
18072024-08-26T20:32:40.264ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 15.355013ms iteration = 27 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
18082024-08-26T20:32:40.981ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 208 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 9f13a8a9-fa11-4898-b9a3-759706c0edc1 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
18092024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 10 reason = Timeout
18102024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 28 reason = Timeout
18112024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
18122024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
18132024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 11 reason = Timeout
18142024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 10 reason = Timeout
18152024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
18162024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 10 reason = Timeout
18172024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
18182024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
18192024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 10 reason = Timeout
18202024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 10 reason = Timeout
18212024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
18222024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 10 reason = Timeout
18232024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 10 reason = Timeout
18242024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
18252024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
18262024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
18272024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
18282024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
18292024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
18302024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
18312024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
18322024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
18332024-08-26T20:32:50.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
18342024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 10 reason = Timeout
18352024-08-26T20:32:50.249ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task started background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:417
18362024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 10 reason = Timeout
18372024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
18382024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
18392024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 10 reason = Timeout
18402024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
18412024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 10 reason = Timeout
18422024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
18432024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
18442024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 10 reason = Timeout
18452024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
18462024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 28 reason = Timeout
18472024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
18482024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 10 reason = Timeout
18492024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
18502024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
18512024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
18522024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
18532024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
18542024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
18552024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
18562024-08-26T20:32:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
18572024-08-26T20:32:50.250ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
18582024-08-26T20:32:50.250ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
18592024-08-26T20:32:50.250ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
18602024-08-26T20:32:50.251ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 2.675255ms iteration = 10 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
18612024-08-26T20:32:50.251ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
18622024-08-26T20:32:50.251ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1069, tv_nsec: 426083665 } })
18632024-08-26T20:32:50.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
18642024-08-26T20:32:50.252ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
18652024-08-26T20:32:50.252ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 3.051629ms iteration = 10 status = {"errors":[],"garbage_collect_requested":[]}
18662024-08-26T20:32:50.252ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
18672024-08-26T20:32:50.252ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1050
18682024-08-26T20:32:50.252ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1069, tv_nsec: 427554719 } })
18692024-08-26T20:32:50.253ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
18702024-08-26T20:32:50.253ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1050
18712024-08-26T20:32:50.253ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
18722024-08-26T20:32:50.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
18732024-08-26T20:32:50.253ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
18742024-08-26T20:32:50.253ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
18752024-08-26T20:32:50.253ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
18762024-08-26T20:32:50.253ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
18772024-08-26T20:32:50.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
18782024-08-26T20:32:50.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
18792024-08-26T20:32:50.254ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 5.769754ms iteration = 10 status = {"region_replacement_started_err":0,"region_replacement_started_ok":0}
18802024-08-26T20:32:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
18812024-08-26T20:32:50.254ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
18822024-08-26T20:32:50.254ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
18832024-08-26T20:32:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
18842024-08-26T20:32:50.255ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
18852024-08-26T20:32:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
18862024-08-26T20:32:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
18872024-08-26T20:32:50.255ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
18882024-08-26T20:32:50.255ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1069, tv_nsec: 430042402 } })
18892024-08-26T20:32:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
18902024-08-26T20:32:50.255ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
18912024-08-26T20:32:50.255ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
18922024-08-26T20:32:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
18932024-08-26T20:32:50.255ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
18942024-08-26T20:32:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
18952024-08-26T20:32:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
18962024-08-26T20:32:50.256ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 7.073527ms iteration = 10 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
18972024-08-26T20:32:50.256ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
18982024-08-26T20:32:50.256ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
18992024-08-26T20:32:50.256ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
19002024-08-26T20:32:50.256ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1050 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
19012024-08-26T20:32:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
19022024-08-26T20:32:50.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
19032024-08-26T20:32:50.257ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
19042024-08-26T20:32:50.257ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
19052024-08-26T20:32:50.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
19062024-08-26T20:32:50.257ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
19072024-08-26T20:32:50.257ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 7.892483ms iteration = 28 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
19082024-08-26T20:32:50.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
19092024-08-26T20:32:50.257ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
19102024-08-26T20:32:50.257ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
19112024-08-26T20:32:50.257ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
19122024-08-26T20:32:50.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
19132024-08-26T20:32:50.257ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
19142024-08-26T20:32:50.258ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
19152024-08-26T20:32:50.258ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task done background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:439
19162024-08-26T20:32:50.258ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 9.176386ms iteration = 10 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
19172024-08-26T20:32:50.259ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
19182024-08-26T20:32:50.259ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
19192024-08-26T20:32:50.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
19202024-08-26T20:32:50.260ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:454 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
19212024-08-26T20:32:50.260ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 11.225655ms iteration = 10 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
19222024-08-26T20:32:50.260ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
19232024-08-26T20:32:50.260ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
19242024-08-26T20:32:50.260ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
19252024-08-26T20:32:50.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
19262024-08-26T20:32:50.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
19272024-08-26T20:32:50.261ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
19282024-08-26T20:32:50.261ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
19292024-08-26T20:32:50.261ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
19302024-08-26T20:32:50.262ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 14.601027ms iteration = 11 status = {"physical_disks_added":0}
19312024-08-26T20:32:50.267ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
19322024-08-26T20:32:50.267ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 18.818296ms iteration = 28 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
19332024-08-26T20:32:50.267ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
19342024-08-26T20:32:50.267ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
19352024-08-26T20:32:50.268ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
19362024-08-26T20:32:50.268ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
19372024-08-26T20:32:50.268ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
19382024-08-26T20:32:50.269ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
19392024-08-26T20:32:50.269ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
19402024-08-26T20:32:50.270ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
19412024-08-26T20:32:50.270ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
19422024-08-26T20:32:50.270ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
19432024-08-26T20:32:50.271ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
19442024-08-26T20:32:50.271ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
19452024-08-26T20:32:50.272ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
19462024-08-26T20:32:50.272ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
19472024-08-26T20:32:50.273ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
19482024-08-26T20:32:50.273ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: c9d3ee27-fe3f-422b-93ce-695a8356d9d2, lookup_type: ById(c9d3ee27-fe3f-422b-93ce-695a8356d9d2) } result = Ok(())
19492024-08-26T20:32:50.273ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
19502024-08-26T20:32:50.273ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
19512024-08-26T20:32:50.274ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 26.169655ms iteration = 10 status = 0
19522024-08-26T20:32:50.287ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
19532024-08-26T20:32:50.287ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:220 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
19542024-08-26T20:32:50.287ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 38.43396ms iteration = 10 status = {"destroyed_active_vmms":0,"error":null,"saga_start_failures":0,"sagas_completed":0,"sagas_failed":0,"sagas_started":0,"terminated_active_migrations":0}
19552024-08-26T20:32:50.417ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
19562024-08-26T20:32:50.417ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
19572024-08-26T20:32:50.417ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "4a1d3ef2-c12d-4a15-b416-0e14f69efd61", "content-length": "26", "date": "Mon, 26 Aug 2024 20:32:50 GMT"} })
19582024-08-26T20:32:50.417ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
19592024-08-26T20:32:50.418ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
19602024-08-26T20:32:50.435ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
19612024-08-26T20:32:50.436ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "0944f954-52df-419d-aedc-cce8122e802a", "content-length": "372", "date": "Mon, 26 Aug 2024 20:32:49 GMT"} })
19622024-08-26T20:32:50.436ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
19632024-08-26T20:32:50.436ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
19642024-08-26T20:32:50.438ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
19652024-08-26T20:32:50.438ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
19662024-08-26T20:32:50.439ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
19672024-08-26T20:32:50.440ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
19682024-08-26T20:32:50.441ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
19692024-08-26T20:32:50.442ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 204, headers: {"x-request-id": "da5b03f6-5b31-4674-9e23-a94ba5778fc8", "date": "Mon, 26 Aug 2024 20:32:49 GMT"} })
19702024-08-26T20:32:50.443ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 194.110666ms iteration = 10 status = {}
19712024-08-26T20:32:50.445ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
19722024-08-26T20:32:50.445ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/v2p", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "ae9536f3-c275-4a36-85b4-949be2e243a5", "content-length": "691", "date": "Mon, 26 Aug 2024 20:32:49 GMT"} })
19732024-08-26T20:32:50.445ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = sock
19742024-08-26T20:32:50.445ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = sock
19752024-08-26T20:32:50.445ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = sock
19762024-08-26T20:32:50.446ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 198.581307ms iteration = 10 status = {}
19772024-08-26T20:32:50.482ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
19782024-08-26T20:32:50.482ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
19792024-08-26T20:32:50.482ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "4d9c7d7c-7d69-4262-b367-2280ed87e996", "content-length": "26", "date": "Mon, 26 Aug 2024 20:32:50 GMT"} })
19802024-08-26T20:32:50.482ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
19812024-08-26T20:32:50.484ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
19822024-08-26T20:32:50.497ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1058 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 zone_address = fd00:1122:3344:101::2
19832024-08-26T20:32:50.497ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
19842024-08-26T20:32:50.498ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "da4dde9b-764a-416a-bf92-c74ec7a8587c", "content-length": "26", "date": "Mon, 26 Aug 2024 20:32:50 GMT"} })
19852024-08-26T20:32:50.498ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1063 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
19862024-08-26T20:32:50.500ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
19872024-08-26T20:32:50.543ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 294.719056ms iteration = 10 status = {}
19882024-08-26T20:32:50.552ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
19892024-08-26T20:32:50.552ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "5ed5508f-a196-4f3e-81d1-4e34467eb006", "content-length": "1", "date": "Mon, 26 Aug 2024 20:32:50 GMT"} })
19902024-08-26T20:32:50.553ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
19912024-08-26T20:32:50.553ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
19922024-08-26T20:32:50.554ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 306.743969ms iteration = 10 status = null
19932024-08-26T20:32:50.625ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
19942024-08-26T20:32:50.625ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
19952024-08-26T20:32:50.627ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:147 port = SwitchPort { id: 77948b01-64a9-403e-be7b-4ef5e426dda4, rack_id: 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(735476d6-05ca-4fbd-a182-ab082f700355) } rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
19962024-08-26T20:32:50.627ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
19972024-08-26T20:32:50.628ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
19982024-08-26T20:32:50.634ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
19992024-08-26T20:32:50.634ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/port/qsfp0/settings", query: Some("tag=omicron"), fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "e02ff728-6dd3-4af3-bda0-5a88b413f7a7", "content-length": "121", "date": "Mon, 26 Aug 2024 20:32:50 GMT"} })
20002024-08-26T20:32:50.635ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1714 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.2.174], params: LinkCreate { autoneg: false, fec: None, kr: false, lane: Some(LinkId(0)), speed: Speed40G } }} } switch_port_id = "qsfp0"
20012024-08-26T20:32:50.635ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1741 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = 735476d6-05ca-4fbd-a182-ab082f700355
20022024-08-26T20:32:50.635ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:379 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
20032024-08-26T20:32:50.635ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
20042024-08-26T20:32:50.635ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:392 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
20052024-08-26T20:32:50.635ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:398 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
20062024-08-26T20:32:50.635ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:418 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
20072024-08-26T20:32:50.635ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
20082024-08-26T20:32:50.635ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
20092024-08-26T20:32:50.637ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
20102024-08-26T20:32:50.637ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv4", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "eba9c3df-dceb-4162-84a6-a37945b792ae", "content-length": "2", "date": "Mon, 26 Aug 2024 20:32:50 GMT"} })
20112024-08-26T20:32:50.637ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
20122024-08-26T20:32:50.638ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv6", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "50cfcab9-84ba-4372-8bcc-1995f491af17", "content-length": "52", "date": "Mon, 26 Aug 2024 20:32:50 GMT"} })
20132024-08-26T20:32:50.638ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0" }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:467 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
20142024-08-26T20:32:50.638ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
20152024-08-26T20:32:50.725ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/switch-ports", query: None, fragment: None }, status: 204, headers: {"x-request-id": "ea8c6273-4928-4784-8f82-12aac8da0db3", "date": "Mon, 26 Aug 2024 20:32:50 GMT"} })
20162024-08-26T20:32:50.725ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:770 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch = Switch0
20172024-08-26T20:32:50.725ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
20182024-08-26T20:32:50.726ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/network-bootstore-config", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "39efc1fd-016c-42f2-bac9-f4f00d8560f9", "content-length": "667", "date": "Mon, 26 Aug 2024 20:32:50 GMT"} })
20192024-08-26T20:32:50.726ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
20202024-08-26T20:32:50.726ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
20212024-08-26T20:32:50.729ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
20222024-08-26T20:32:50.729ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
20232024-08-26T20:32:50.731ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
20242024-08-26T20:32:50.732ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
20252024-08-26T20:32:50.733ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
20262024-08-26T20:32:50.733ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
20272024-08-26T20:32:50.739ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 491.408406ms iteration = 10 status = {}
20282024-08-26T20:32:50.981ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 219 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 0340fa37-d85b-40a5-879f-8364046ee4c4 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
20292024-08-26T20:33:00.248ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 29 reason = Timeout
20302024-08-26T20:33:00.248ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
20312024-08-26T20:33:00.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
20322024-08-26T20:33:00.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 29 reason = Timeout
20332024-08-26T20:33:00.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
20342024-08-26T20:33:00.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
20352024-08-26T20:33:00.250ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
20362024-08-26T20:33:00.250ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
20372024-08-26T20:33:00.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
20382024-08-26T20:33:00.251ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 2.842947ms iteration = 29 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
20392024-08-26T20:33:00.252ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
20402024-08-26T20:33:00.252ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
20412024-08-26T20:33:00.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
20422024-08-26T20:33:00.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
20432024-08-26T20:33:00.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
20442024-08-26T20:33:00.255ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
20452024-08-26T20:33:00.257ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
20462024-08-26T20:33:00.257ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
20472024-08-26T20:33:00.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
20482024-08-26T20:33:00.263ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
20492024-08-26T20:33:00.263ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 15.345123ms iteration = 29 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
20502024-08-26T20:33:00.981ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 216 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 48352a09-18c3-4d8a-a2d9-3aab1af27990 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
20512024-08-26T20:33:10.248ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 30 reason = Timeout
20522024-08-26T20:33:10.248ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
20532024-08-26T20:33:10.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
20542024-08-26T20:33:10.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 30 reason = Timeout
20552024-08-26T20:33:10.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
20562024-08-26T20:33:10.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
20572024-08-26T20:33:10.250ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
20582024-08-26T20:33:10.250ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
20592024-08-26T20:33:10.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
20602024-08-26T20:33:10.251ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 2.670134ms iteration = 30 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
20612024-08-26T20:33:10.252ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
20622024-08-26T20:33:10.252ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
20632024-08-26T20:33:10.252ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
20642024-08-26T20:33:10.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
20652024-08-26T20:33:10.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
20662024-08-26T20:33:10.255ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
20672024-08-26T20:33:10.257ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
20682024-08-26T20:33:10.257ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
20692024-08-26T20:33:10.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
20702024-08-26T20:33:10.265ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
20712024-08-26T20:33:10.265ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 16.500764ms iteration = 30 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
20722024-08-26T20:33:10.981ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 239 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = d09c9235-3ca2-43a8-8649-c18cfde6a722 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
20732024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 8 reason = Timeout
20742024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 6 reason = Timeout
20752024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 12 reason = Timeout
20762024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 7 reason = Timeout
20772024-08-26T20:33:20.249ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:80 reason = blueprint disabled target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460
20782024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
20792024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 20.18\u{b5}s iteration = 7 status = {"error":"blueprint disabled","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460"}
20802024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 31 reason = Timeout
20812024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 6 reason = Timeout
20822024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
20832024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
20842024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
20852024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 6 reason = Timeout
20862024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 11 reason = Timeout
20872024-08-26T20:33:20.249ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
20882024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
20892024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
20902024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 11 reason = Timeout
20912024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
20922024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 11 reason = Timeout
20932024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
20942024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 6 reason = Timeout
20952024-08-26T20:33:20.249ZINFOnexus (BackgroundTasks): Abandoned VMM reaper running background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:76
20962024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
20972024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = service_firewall_rule_propagation iteration = 2 reason = Timeout
20982024-08-26T20:33:20.249ZDEBGnexus (service-firewall-rule-progator): starting background task for service firewall rule propagation background_task = service_firewall_rule_propagation
20992024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
21002024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 6 reason = Timeout
21012024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
21022024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 6 reason = Timeout
21032024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
21042024-08-26T20:33:20.249ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-08-26 20:23:20.249358934 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
21052024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
21062024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 6 reason = Timeout
21072024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
21082024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 11 reason = Timeout
21092024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 31 reason = Timeout
21102024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
21112024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
21122024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 11 reason = Timeout
21132024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
21142024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 11 reason = Timeout
21152024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
21162024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
21172024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 11 reason = Timeout
21182024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 6 reason = Timeout
21192024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 11 reason = Timeout
21202024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
21212024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 8 reason = Timeout
21222024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 11 reason = Timeout
21232024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 11 reason = Timeout
21242024-08-26T20:33:20.249ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task started background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:417
21252024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
21262024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
21272024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 11 reason = Timeout
21282024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
21292024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 6 reason = Timeout
21302024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
21312024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 11 reason = Timeout
21322024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
21332024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 11 reason = Timeout
21342024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
21352024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
21362024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
21372024-08-26T20:33:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
21382024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
21392024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
21402024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
21412024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
21422024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
21432024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
21442024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
21452024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
21462024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
21472024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
21482024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
21492024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
21502024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
21512024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
21522024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
21532024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
21542024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
21552024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
21562024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
21572024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
21582024-08-26T20:33:20.250ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("be00167b-db1f-4cc1-86f1-a2af7f1c31f8.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("d2b5c33b-9149-4357-86b1-60e922261ec4.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("d2b5c33b-9149-4357-86b1-60e922261ec4.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 1099, tv_nsec: 424829793 } })
21592024-08-26T20:33:20.250ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("316f64f2-8fa5-43f3-8d5d-92212e3046c4.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("35cb7d67-b9f1-4dcf-bac3-28244fb25d88.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("e37633b8-b58b-40c2-b3c2-6873358565b7.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("e37633b8-b58b-40c2-b3c2-6873358565b7.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:3::1))) }], valid_until: Instant { tv_sec: 1099, tv_nsec: 424902354 } })
21602024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
21612024-08-26T20:33:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
21622024-08-26T20:33:20.252ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
21632024-08-26T20:33:20.252ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 2.958877ms iteration = 6 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
21642024-08-26T20:33:20.252ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
21652024-08-26T20:33:20.252ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
21662024-08-26T20:33:20.252ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
21672024-08-26T20:33:20.252ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } dns_group = internal
21682024-08-26T20:33:20.252ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 3.326301ms iteration = 6 status = {"addresses":["[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353","[fd00:1122:3344:3::1]:5353"]}
21692024-08-26T20:33:20.253ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
21702024-08-26T20:33:20.253ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.788235ms iteration = 11 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
21712024-08-26T20:33:20.253ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
21722024-08-26T20:33:20.253ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
21732024-08-26T20:33:20.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
21742024-08-26T20:33:20.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
21752024-08-26T20:33:20.254ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
21762024-08-26T20:33:20.254ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
21772024-08-26T20:33:20.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
21782024-08-26T20:33:20.254ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
21792024-08-26T20:33:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
21802024-08-26T20:33:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
21812024-08-26T20:33:20.255ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
21822024-08-26T20:33:20.255ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
21832024-08-26T20:33:20.255ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
21842024-08-26T20:33:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
21852024-08-26T20:33:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
21862024-08-26T20:33:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
21872024-08-26T20:33:20.256ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
21882024-08-26T20:33:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
21892024-08-26T20:33:20.256ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
21902024-08-26T20:33:20.256ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
21912024-08-26T20:33:20.256ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
21922024-08-26T20:33:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
21932024-08-26T20:33:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
21942024-08-26T20:33:20.257ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 7.861814ms iteration = 11 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
21952024-08-26T20:33:20.257ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
21962024-08-26T20:33:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
21972024-08-26T20:33:20.257ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
21982024-08-26T20:33:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
21992024-08-26T20:33:20.257ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
22002024-08-26T20:33:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
22012024-08-26T20:33:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
22022024-08-26T20:33:20.258ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
22032024-08-26T20:33:20.258ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
22042024-08-26T20:33:20.258ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
22052024-08-26T20:33:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
22062024-08-26T20:33:20.258ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
22072024-08-26T20:33:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
22082024-08-26T20:33:20.259ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
22092024-08-26T20:33:20.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
22102024-08-26T20:33:20.259ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
22112024-08-26T20:33:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
22122024-08-26T20:33:20.260ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
22132024-08-26T20:33:20.260ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1099, tv_nsec: 434449003 } })
22142024-08-26T20:33:20.260ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
22152024-08-26T20:33:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
22162024-08-26T20:33:20.260ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1050
22172024-08-26T20:33:20.261ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
22182024-08-26T20:33:20.261ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
22192024-08-26T20:33:20.261ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
22202024-08-26T20:33:20.261ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
22212024-08-26T20:33:20.264ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
22222024-08-26T20:33:20.264ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
22232024-08-26T20:33:20.264ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
22242024-08-26T20:33:20.264ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
22252024-08-26T20:33:20.264ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
22262024-08-26T20:33:20.270ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
22272024-08-26T20:33:20.270ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
22282024-08-26T20:33:20.271ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
22292024-08-26T20:33:20.271ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
22302024-08-26T20:33:20.271ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
22312024-08-26T20:33:20.271ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
22322024-08-26T20:33:20.271ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
22332024-08-26T20:33:20.271ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
22342024-08-26T20:33:20.272ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
22352024-08-26T20:33:20.272ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
22362024-08-26T20:33:20.272ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
22372024-08-26T20:33:20.273ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
22382024-08-26T20:33:20.273ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
22392024-08-26T20:33:20.274ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task done background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:439
22402024-08-26T20:33:20.274ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 24.882183ms iteration = 11 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
22412024-08-26T20:33:20.274ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
22422024-08-26T20:33:20.274ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
22432024-08-26T20:33:20.275ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
22442024-08-26T20:33:20.275ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
22452024-08-26T20:33:20.275ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
22462024-08-26T20:33:20.275ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
22472024-08-26T20:33:20.275ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
22482024-08-26T20:33:20.275ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
22492024-08-26T20:33:20.276ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
22502024-08-26T20:33:20.276ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
22512024-08-26T20:33:20.276ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
22522024-08-26T20:33:20.276ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
22532024-08-26T20:33:20.276ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
22542024-08-26T20:33:20.276ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
22552024-08-26T20:33:20.277ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
22562024-08-26T20:33:20.277ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
22572024-08-26T20:33:20.277ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
22582024-08-26T20:33:20.278ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
22592024-08-26T20:33:20.278ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
22602024-08-26T20:33:20.278ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
22612024-08-26T20:33:20.278ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
22622024-08-26T20:33:20.278ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
22632024-08-26T20:33:20.278ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
22642024-08-26T20:33:20.278ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
22652024-08-26T20:33:20.279ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1099, tv_nsec: 453453201 } })
22662024-08-26T20:33:20.279ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
22672024-08-26T20:33:20.279ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
22682024-08-26T20:33:20.280ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1050
22692024-08-26T20:33:20.280ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
22702024-08-26T20:33:20.280ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 30.792258ms iteration = 11 status = {"region_replacement_started_err":0,"region_replacement_started_ok":0}
22712024-08-26T20:33:20.281ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
22722024-08-26T20:33:20.281ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
22732024-08-26T20:33:20.281ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
22742024-08-26T20:33:20.281ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
22752024-08-26T20:33:20.282ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
22762024-08-26T20:33:20.283ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
22772024-08-26T20:33:20.283ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = d840a78c-2f0d-4cdf-9962-31a3dae384a2 dns_zone_name = oxide.test found_so_far = 0 version = 2
22782024-08-26T20:33:20.283ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
22792024-08-26T20:33:20.283ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
22802024-08-26T20:33:20.283ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
22812024-08-26T20:33:20.283ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
22822024-08-26T20:33:20.283ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
22832024-08-26T20:33:20.283ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
22842024-08-26T20:33:20.284ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
22852024-08-26T20:33:20.284ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 35.27086ms iteration = 31 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
22862024-08-26T20:33:20.284ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
22872024-08-26T20:33:20.284ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
22882024-08-26T20:33:20.284ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:454 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
22892024-08-26T20:33:20.284ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 35.554052ms iteration = 11 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
22902024-08-26T20:33:20.285ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
22912024-08-26T20:33:20.285ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = b82a4171-23b2-444b-9597-09085a7a2fb8 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
22922024-08-26T20:33:20.285ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
22932024-08-26T20:33:20.286ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
22942024-08-26T20:33:20.286ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
22952024-08-26T20:33:20.286ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
22962024-08-26T20:33:20.286ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-08-26 20:23:20.249358934 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
22972024-08-26T20:33:20.286ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 36.779224ms iteration = 6 status = {"expiration":"2024-08-26T20:23:20.249358934Z","pruned":[]}
22982024-08-26T20:33:20.286ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
22992024-08-26T20:33:20.286ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
23002024-08-26T20:33:20.286ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
23012024-08-26T20:33:20.286ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
23022024-08-26T20:33:20.286ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
23032024-08-26T20:33:20.286ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
23042024-08-26T20:33:20.286ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
23052024-08-26T20:33:20.286ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
23062024-08-26T20:33:20.287ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
23072024-08-26T20:33:20.287ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 37.905614ms iteration = 6 status = {"errors":[],"found_port_ok":[]}
23082024-08-26T20:33:20.287ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
23092024-08-26T20:33:20.287ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
23102024-08-26T20:33:20.288ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
23112024-08-26T20:33:20.288ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
23122024-08-26T20:33:20.288ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
23132024-08-26T20:33:20.288ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
23142024-08-26T20:33:20.288ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
23152024-08-26T20:33:20.288ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
23162024-08-26T20:33:20.288ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
23172024-08-26T20:33:20.288ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
23182024-08-26T20:33:20.289ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
23192024-08-26T20:33:20.289ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1099, tv_nsec: 463535445 } })
23202024-08-26T20:33:20.289ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = d840a78c-2f0d-4cdf-9962-31a3dae384a2 dns_zone_name = oxide.test found_so_far = 1 version = 2
23212024-08-26T20:33:20.289ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
23222024-08-26T20:33:20.289ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-08-26 20:29:01.487349 UTC dns_group = external generation = 2
23232024-08-26T20:33:20.289ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 40.196036ms iteration = 6 status = {"generation":2}
23242024-08-26T20:33:20.289ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
23252024-08-26T20:33:20.289ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
23262024-08-26T20:33:20.289ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
23272024-08-26T20:33:20.290ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1050 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
23282024-08-26T20:33:20.290ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
23292024-08-26T20:33:20.290ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
23302024-08-26T20:33:20.290ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
23312024-08-26T20:33:20.290ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
23322024-08-26T20:33:20.290ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
23332024-08-26T20:33:20.290ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
23342024-08-26T20:33:20.290ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
23352024-08-26T20:33:20.290ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
23362024-08-26T20:33:20.291ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
23372024-08-26T20:33:20.291ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
23382024-08-26T20:33:20.291ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 41.760511ms iteration = 11 status = {"errors":[],"garbage_collect_requested":[]}
23392024-08-26T20:33:20.291ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
23402024-08-26T20:33:20.291ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
23412024-08-26T20:33:20.292ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
23422024-08-26T20:33:20.292ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = b82a4171-23b2-444b-9597-09085a7a2fb8 dns_zone_name = control-plane.oxide.internal found_so_far = 50 version = 1
23432024-08-26T20:33:20.292ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
23442024-08-26T20:33:20.292ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-08-26 20:29:01.089496 UTC dns_group = internal generation = 1
23452024-08-26T20:33:20.292ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 43.077233ms iteration = 6 status = {"generation":1}
23462024-08-26T20:33:20.292ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
23472024-08-26T20:33:20.293ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: c9d3ee27-fe3f-422b-93ce-695a8356d9d2, lookup_type: ById(c9d3ee27-fe3f-422b-93ce-695a8356d9d2) } result = Ok(())
23482024-08-26T20:33:20.293ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
23492024-08-26T20:33:20.293ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
23502024-08-26T20:33:20.293ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
23512024-08-26T20:33:20.293ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
23522024-08-26T20:33:20.293ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
23532024-08-26T20:33:20.293ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
23542024-08-26T20:33:20.293ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
23552024-08-26T20:33:20.294ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 44.859129ms iteration = 11 status = 0
23562024-08-26T20:33:20.294ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
23572024-08-26T20:33:20.295ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
23582024-08-26T20:33:20.295ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0
23592024-08-26T20:33:20.295ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:189 found = 0 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0
23602024-08-26T20:33:20.295ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 46.065391ms iteration = 6 status = {"error":null,"error_count":0,"found":0,"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0}
23612024-08-26T20:33:20.296ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
23622024-08-26T20:33:20.296ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
23632024-08-26T20:33:20.296ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
23642024-08-26T20:33:20.296ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 47.295662ms iteration = 6 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
23652024-08-26T20:33:20.297ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
23662024-08-26T20:33:20.297ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
23672024-08-26T20:33:20.297ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
23682024-08-26T20:33:20.297ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
23692024-08-26T20:33:20.299ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
23702024-08-26T20:33:20.299ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:220 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
23712024-08-26T20:33:20.299ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 49.549644ms iteration = 11 status = {"destroyed_active_vmms":0,"error":null,"saga_start_failures":0,"sagas_completed":0,"sagas_failed":0,"sagas_started":0,"terminated_active_migrations":0}
23722024-08-26T20:33:20.300ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
23732024-08-26T20:33:20.300ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
23742024-08-26T20:33:20.300ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
23752024-08-26T20:33:20.301ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
23762024-08-26T20:33:20.301ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
23772024-08-26T20:33:20.302ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
23782024-08-26T20:33:20.303ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
23792024-08-26T20:33:20.304ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
23802024-08-26T20:33:20.304ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
23812024-08-26T20:33:20.304ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-08-26T20:28:18.089116Z, time_modified: 2024-08-26T20:28:18.089116Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 80a72585-f38a-4ccb-b69a-caafab415ca2, db_silo: Silo { identity: SiloIdentity { id: 80a72585-f38a-4ccb-b69a-caafab415ca2, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-08-26T20:29:01.450089Z, time_modified: 2024-08-26T20:29:01.450089Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 80a72585-f38a-4ccb-b69a-caafab415ca2, db_silo: Silo { identity: SiloIdentity { id: 80a72585-f38a-4ccb-b69a-caafab415ca2, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-08-26T20:29:01.450089Z, time_modified: 2024-08-26T20:29:01.450089Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923" }] }) }
23822024-08-26T20:33:20.305ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 55.59673ms iteration = 6 status = {"by_dns_name":{"default-silo.sys.oxide.test":{"silo_id":"001de000-5110-4000-8000-000000000000","tls_certs":[]},"recovery.sys.oxide.test":{"silo_id":"80a72585-f38a-4ccb-b69a-caafab415ca2","tls_certs":["81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923"]}},"default_endpoint":{"silo_id":"80a72585-f38a-4ccb-b69a-caafab415ca2","tls_certs":["81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923"]},"warnings":["silo 001de000-5110-4000-8000-000000000000 with DNS name \\"default-silo.sys.oxide.test\\" has no usable certificates"]}
23832024-08-26T20:33:20.305ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
23842024-08-26T20:33:20.305ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
23852024-08-26T20:33:20.306ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
23862024-08-26T20:33:20.306ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
23872024-08-26T20:33:20.307ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
23882024-08-26T20:33:20.307ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
23892024-08-26T20:33:20.308ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
23902024-08-26T20:33:20.311ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000001, lookup_type: ByName("external-dns") } result = Ok(())
23912024-08-26T20:33:20.311ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
23922024-08-26T20:33:20.311ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
23932024-08-26T20:33:20.312ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
23942024-08-26T20:33:20.313ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
23952024-08-26T20:33:20.314ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
23962024-08-26T20:33:20.314ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
23972024-08-26T20:33:20.315ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
23982024-08-26T20:33:20.319ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000001, lookup_type: ByName("external-dns") } result = Ok(())
23992024-08-26T20:33:20.319ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
24002024-08-26T20:33:20.319ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
24012024-08-26T20:33:20.321ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 72.629709ms iteration = 12 status = {"physical_disks_added":0}
24022024-08-26T20:33:20.322ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
24032024-08-26T20:33:20.322ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
24042024-08-26T20:33:20.323ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
24052024-08-26T20:33:20.323ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
24062024-08-26T20:33:20.325ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
24072024-08-26T20:33:20.325ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
24082024-08-26T20:33:20.326ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
24092024-08-26T20:33:20.326ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
24102024-08-26T20:33:20.327ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
24112024-08-26T20:33:20.327ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
24122024-08-26T20:33:20.328ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
24132024-08-26T20:33:20.329ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
24142024-08-26T20:33:20.330ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
24152024-08-26T20:33:20.330ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
24162024-08-26T20:33:20.331ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
24172024-08-26T20:33:20.335ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000002, lookup_type: ByName("nexus") } result = Ok(())
24182024-08-26T20:33:20.335ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
24192024-08-26T20:33:20.336ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
24202024-08-26T20:33:20.337ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
24212024-08-26T20:33:20.337ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
24222024-08-26T20:33:20.338ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
24232024-08-26T20:33:20.338ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
24242024-08-26T20:33:20.340ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
24252024-08-26T20:33:20.344ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000002, lookup_type: ByName("nexus") } result = Ok(())
24262024-08-26T20:33:20.344ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
24272024-08-26T20:33:20.344ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
24282024-08-26T20:33:20.348ZDEBGnexus (service-firewall-rule-progator): resolved names for firewall rules background_task = service_firewall_rule_propagation instance_interfaces = {} subnet_interfaces = {Name("nexus"): [NetworkInterface { id: 51b5de96-7858-46f9-a3f6-2973be29eca4, kind: Service { id: a16170da-94b1-410d-ba50-70fcdfd6415c }, name: Name("nexus-a16170da-94b1-410d-ba50-70fcdfd6415c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 194])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b4a6a762-4058-4ca6-b80d-4310017570b5, kind: Service { id: 35c8e192-84c1-4647-9597-46bc9b76fa72 }, name: Name("nexus-35c8e192-84c1-4647-9597-46bc9b76fa72"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 142, 172])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c97fa9f6-15f2-4316-b5a2-63193958d1c4, kind: Service { id: ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5 }, name: Name("nexus-ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 109])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], Name("external-dns"): [NetworkInterface { id: dc631b0f-a898-4f09-be46-9f62f3a19801, kind: Service { id: d2b5c33b-9149-4357-86b1-60e922261ec4 }, name: Name("external-dns-d2b5c33b-9149-4357-86b1-60e922261ec4"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f0a22b00-eade-4b70-8547-e66590faf4f9, kind: Service { id: be00167b-db1f-4cc1-86f1-a2af7f1c31f8 }, name: Name("external-dns-be00167b-db1f-4cc1-86f1-a2af7f1c31f8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 129, 0])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }]} subnet_networks = {Name("nexus"): [V4(Ipv4Network { addr: 172.30.2.0, prefix: 24 }), V6(Ipv6Network { addr: fd77:e9d2:9cd9:2::, prefix: 64 })], Name("external-dns"): [V4(Ipv4Network { addr: 172.30.1.0, prefix: 24 }), V6(Ipv6Network { addr: fd77:e9d2:9cd9:1::, prefix: 64 })]} vpc_interfaces = {}
24292024-08-26T20:33:20.348ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
24302024-08-26T20:33:20.349ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
24312024-08-26T20:33:20.350ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
24322024-08-26T20:33:20.352ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Fleet result = Ok(())
24332024-08-26T20:33:20.352ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
24342024-08-26T20:33:20.352ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
24352024-08-26T20:33:20.353ZDEBGnexus (service-firewall-rule-progator): Allowlist for user-facing services is set to allow any inbound traffic. Existing VPC firewall rules will not be modified. background_task = service_firewall_rule_propagation
24362024-08-26T20:33:20.353ZDEBGnexus (service-firewall-rule-progator): resolved firewall rules for sled agents background_task = service_firewall_rule_propagation sled_agent_rules = [ResolvedVpcFirewallRule { action: Allow, direction: Inbound, filter_hosts: None, filter_ports: Some([L4PortRange("53")]), filter_protocols: Some([Udp]), priority: 65534, status: Enabled, targets: [NetworkInterface { id: dc631b0f-a898-4f09-be46-9f62f3a19801, kind: Service { id: d2b5c33b-9149-4357-86b1-60e922261ec4 }, name: Name("external-dns-d2b5c33b-9149-4357-86b1-60e922261ec4"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f0a22b00-eade-4b70-8547-e66590faf4f9, kind: Service { id: be00167b-db1f-4cc1-86f1-a2af7f1c31f8 }, name: Name("external-dns-be00167b-db1f-4cc1-86f1-a2af7f1c31f8"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 129, 0])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }] }, ResolvedVpcFirewallRule { action: Allow, direction: Inbound, filter_hosts: None, filter_ports: Some([L4PortRange("80"), L4PortRange("443")]), filter_protocols: Some([Tcp]), priority: 65534, status: Enabled, targets: [NetworkInterface { id: 51b5de96-7858-46f9-a3f6-2973be29eca4, kind: Service { id: a16170da-94b1-410d-ba50-70fcdfd6415c }, name: Name("nexus-a16170da-94b1-410d-ba50-70fcdfd6415c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 194])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b4a6a762-4058-4ca6-b80d-4310017570b5, kind: Service { id: 35c8e192-84c1-4647-9597-46bc9b76fa72 }, name: Name("nexus-35c8e192-84c1-4647-9597-46bc9b76fa72"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 142, 172])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c97fa9f6-15f2-4316-b5a2-63193958d1c4, kind: Service { id: ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5 }, name: Name("nexus-ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 109])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }] }]
24372024-08-26T20:33:20.353ZDEBGnexus (service-firewall-rule-progator): resolved 2 rules for sleds background_task = service_firewall_rule_propagation
24382024-08-26T20:33:20.376ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
24392024-08-26T20:33:20.404ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
24402024-08-26T20:33:20.422ZDEBGnexus (service-firewall-rule-progator): resolved sleds for vpc oxide-services background_task = service_firewall_rule_propagation vpc_to_sled = [Sled { identity: SledIdentity { id: c9d3ee27-fe3f-422b-93ce-695a8356d9d2, time_created: 2024-08-26T20:28:58.817135Z, time_modified: 2024-08-26T20:28:58.817135Z }, time_deleted: None, rcgen: Generation(Generation(19)), rack_id: 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29, is_scrimlet: true, serial_number: "sock", part_number: "i86pc", revision: SqlU32(0), usable_hardware_threads: SqlU32(48), usable_physical_ram: ByteCount(ByteCount(102952026112)), reservoir_size: ByteCount(ByteCount(51474595840)), ip: fd00:1122:3344:101::1, port: SqlU16(12345), last_used_address: fd00:1122:3344:101::ffff, policy: InService, state: Active, sled_agent_gen: Generation(Generation(1)) }]
24412024-08-26T20:33:20.422ZDEBGnexus (service-firewall-rule-progator): sending firewall rules to sled agents background_task = service_firewall_rule_propagation
24422024-08-26T20:33:20.422ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
24432024-08-26T20:33:20.422ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
24442024-08-26T20:33:20.423ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
24452024-08-26T20:33:20.423ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
24462024-08-26T20:33:20.425ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
24472024-08-26T20:33:20.427ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Sled { parent: Fleet, key: c9d3ee27-fe3f-422b-93ce-695a8356d9d2, lookup_type: ById(c9d3ee27-fe3f-422b-93ce-695a8356d9d2) } result = Ok(())
24482024-08-26T20:33:20.454ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
24492024-08-26T20:33:20.454ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
24502024-08-26T20:33:20.454ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "acf652bf-405f-41b7-b97e-be3e1d593ce0", "content-length": "26", "date": "Mon, 26 Aug 2024 20:33:20 GMT"} })
24512024-08-26T20:33:20.454ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
24522024-08-26T20:33:20.457ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
24532024-08-26T20:33:20.462ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
24542024-08-26T20:33:20.462ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "5056081d-8b96-45a1-885b-7af5fccb119e", "content-length": "372", "date": "Mon, 26 Aug 2024 20:33:19 GMT"} })
24552024-08-26T20:33:20.462ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
24562024-08-26T20:33:20.462ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
24572024-08-26T20:33:20.463ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
24582024-08-26T20:33:20.463ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
24592024-08-26T20:33:20.463ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
24602024-08-26T20:33:20.464ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
24612024-08-26T20:33:20.464ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 214.816219ms iteration = 31 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
24622024-08-26T20:33:20.466ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
24632024-08-26T20:33:20.466ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
24642024-08-26T20:33:20.468ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
24652024-08-26T20:33:20.468ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 204, headers: {"x-request-id": "01a46958-37e4-40dd-adad-647bcaba5cb5", "date": "Mon, 26 Aug 2024 20:33:19 GMT"} })
24662024-08-26T20:33:20.469ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 220.594633ms iteration = 11 status = {}
24672024-08-26T20:33:20.482ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
24682024-08-26T20:33:20.483ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/v2p", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "40694dc2-b3e5-4e1d-b8a9-e67c3e3d34e8", "content-length": "691", "date": "Mon, 26 Aug 2024 20:33:19 GMT"} })
24692024-08-26T20:33:20.483ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = sock
24702024-08-26T20:33:20.483ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = sock
24712024-08-26T20:33:20.483ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = sock
24722024-08-26T20:33:20.484ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 235.734815ms iteration = 11 status = {}
24732024-08-26T20:33:20.499ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1058 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 zone_address = fd00:1122:3344:101::2
24742024-08-26T20:33:20.499ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
24752024-08-26T20:33:20.500ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "331cea94-b2fb-4a87-befa-d15f616aea5d", "content-length": "26", "date": "Mon, 26 Aug 2024 20:33:20 GMT"} })
24762024-08-26T20:33:20.500ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1063 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
24772024-08-26T20:33:20.502ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
24782024-08-26T20:33:20.509ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
24792024-08-26T20:33:20.509ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
24802024-08-26T20:33:20.510ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "1f88dec5-166f-4184-8e29-bd80575f4f4d", "content-length": "26", "date": "Mon, 26 Aug 2024 20:33:20 GMT"} })
24812024-08-26T20:33:20.510ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
24822024-08-26T20:33:20.512ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
24832024-08-26T20:33:20.536ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
24842024-08-26T20:33:20.537ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::8)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "a4e8ee7a-5614-43ff-9256-566338b897c6", "date": "Mon, 26 Aug 2024 20:33:19 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
24852024-08-26T20:33:20.537ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
24862024-08-26T20:33:20.538ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::9)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "c39417ed-08fb-4c97-9e3f-83b5f3861bf7", "date": "Mon, 26 Aug 2024 20:33:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
24872024-08-26T20:33:20.538ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
24882024-08-26T20:33:20.540ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
24892024-08-26T20:33:20.540ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 291.468816ms iteration = 8 status = {"generation":2,"server_results":{"[fd00:1122:3344:101::8]:5353":{"Ok":null},"[fd00:1122:3344:101::9]:5353":{"Ok":null}}}
24902024-08-26T20:33:20.585ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
24912024-08-26T20:33:20.600ZDEBGnexus (service-firewall-rule-progator): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = service_firewall_rule_propagation body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc/001de000-074c-4000-8000-000000000000/firewall/rules
24922024-08-26T20:33:20.603ZDEBGnexus (service-firewall-rule-progator): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = service_firewall_rule_propagation result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc/001de000-074c-4000-8000-000000000000/firewall/rules", query: None, fragment: None }, status: 204, headers: {"x-request-id": "e9026c4a-e3c3-49f7-863f-b4dbaebe273f", "date": "Mon, 26 Aug 2024 20:33:19 GMT"} })
24932024-08-26T20:33:20.605ZINFOnexus (service-firewall-rule-progator): updated firewall rules on 1 sleds background_task = service_firewall_rule_propagation file = nexus/networking/src/firewall_rules.rs:496
24942024-08-26T20:33:20.605ZDEBGnexus (service-firewall-rule-progator): successfully propagated service firewall rules background_task = service_firewall_rule_propagation elapsed = 356.220711ms
24952024-08-26T20:33:20.605ZDEBGnexus (BackgroundTasks): activation complete background_task = service_firewall_rule_propagation elapsed = 356.234531ms iteration = 2 status = {}
24962024-08-26T20:33:20.635ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
24972024-08-26T20:33:20.636ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "1df7a2e6-0731-4f18-b91c-bfd36bdfb4a1", "content-length": "1", "date": "Mon, 26 Aug 2024 20:33:20 GMT"} })
24982024-08-26T20:33:20.638ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
24992024-08-26T20:33:20.638ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
25002024-08-26T20:33:20.639ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 390.125459ms iteration = 11 status = null
25012024-08-26T20:33:20.687ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
25022024-08-26T20:33:20.688ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:2::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "ccaf6797-8c8c-434d-af72-9431989cbb4d", "date": "Mon, 26 Aug 2024 20:33:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
25032024-08-26T20:33:20.688ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
25042024-08-26T20:33:20.690ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:1::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "77954c5f-6327-43b5-a086-35f35e7d0d3b", "date": "Mon, 26 Aug 2024 20:33:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
25052024-08-26T20:33:20.690ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
25062024-08-26T20:33:20.691ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:3::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "fbee5cee-f5af-49c1-9e4e-5a5250345aff", "date": "Mon, 26 Aug 2024 20:33:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
25072024-08-26T20:33:20.691ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
25082024-08-26T20:33:20.692ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 442.364617ms iteration = 11 status = {}
25092024-08-26T20:33:20.693ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
25102024-08-26T20:33:20.693ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 443.466917ms iteration = 8 status = {"generation":1,"server_results":{"[fd00:1122:3344:1::1]:5353":{"Ok":null},"[fd00:1122:3344:2::1]:5353":{"Ok":null},"[fd00:1122:3344:3::1]:5353":{"Ok":null}}}
25112024-08-26T20:33:20.759ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
25122024-08-26T20:33:20.760ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
25132024-08-26T20:33:20.761ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:147 port = SwitchPort { id: 77948b01-64a9-403e-be7b-4ef5e426dda4, rack_id: 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(735476d6-05ca-4fbd-a182-ab082f700355) } rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
25142024-08-26T20:33:20.761ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
25152024-08-26T20:33:20.762ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
25162024-08-26T20:33:20.769ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
25172024-08-26T20:33:20.769ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/port/qsfp0/settings", query: Some("tag=omicron"), fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "25e2594c-bcc6-4ac4-a469-1d1d4462e2ce", "content-length": "121", "date": "Mon, 26 Aug 2024 20:33:20 GMT"} })
25182024-08-26T20:33:20.769ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1714 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.2.174], params: LinkCreate { autoneg: false, fec: None, kr: false, lane: Some(LinkId(0)), speed: Speed40G } }} } switch_port_id = "qsfp0"
25192024-08-26T20:33:20.769ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1741 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = 735476d6-05ca-4fbd-a182-ab082f700355
25202024-08-26T20:33:20.769ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:379 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
25212024-08-26T20:33:20.769ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
25222024-08-26T20:33:20.769ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:392 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
25232024-08-26T20:33:20.769ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:398 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
25242024-08-26T20:33:20.769ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:418 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
25252024-08-26T20:33:20.769ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
25262024-08-26T20:33:20.770ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
25272024-08-26T20:33:20.771ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
25282024-08-26T20:33:20.771ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv4", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "bbb775cf-96b8-4224-b628-f5d0c3890240", "content-length": "2", "date": "Mon, 26 Aug 2024 20:33:20 GMT"} })
25292024-08-26T20:33:20.771ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
25302024-08-26T20:33:20.772ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv6", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "77d2e774-4af8-465e-8619-59e9e38c02c1", "content-length": "52", "date": "Mon, 26 Aug 2024 20:33:20 GMT"} })
25312024-08-26T20:33:20.772ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0" }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:467 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
25322024-08-26T20:33:20.772ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
25332024-08-26T20:33:20.860ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/switch-ports", query: None, fragment: None }, status: 204, headers: {"x-request-id": "dc162002-bb14-4d4e-ac03-50a027fb1eb3", "date": "Mon, 26 Aug 2024 20:33:20 GMT"} })
25342024-08-26T20:33:20.860ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:770 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch = Switch0
25352024-08-26T20:33:20.860ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
25362024-08-26T20:33:20.861ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/network-bootstore-config", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "0fbe3d9d-423c-4e3d-addb-d05db1639b21", "content-length": "667", "date": "Mon, 26 Aug 2024 20:33:20 GMT"} })
25372024-08-26T20:33:20.861ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
25382024-08-26T20:33:20.862ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
25392024-08-26T20:33:20.865ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
25402024-08-26T20:33:20.865ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
25412024-08-26T20:33:20.867ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
25422024-08-26T20:33:20.868ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
25432024-08-26T20:33:20.869ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
25442024-08-26T20:33:20.869ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
25452024-08-26T20:33:20.876ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 626.709361ms iteration = 11 status = {}
25462024-08-26T20:33:20.982ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 238 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 1d7386bc-6d4a-4ad1-9d8f-99c8021aec57 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
25472024-08-26T20:33:30.248ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 32 reason = Timeout
25482024-08-26T20:33:30.248ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
25492024-08-26T20:33:30.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
25502024-08-26T20:33:30.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 32 reason = Timeout
25512024-08-26T20:33:30.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
25522024-08-26T20:33:30.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
25532024-08-26T20:33:30.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
25542024-08-26T20:33:30.251ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
25552024-08-26T20:33:30.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
25562024-08-26T20:33:30.252ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 2.834267ms iteration = 32 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
25572024-08-26T20:33:30.253ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
25582024-08-26T20:33:30.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
25592024-08-26T20:33:30.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
25602024-08-26T20:33:30.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
25612024-08-26T20:33:30.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
25622024-08-26T20:33:30.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
25632024-08-26T20:33:30.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
25642024-08-26T20:33:30.258ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
25652024-08-26T20:33:30.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
25662024-08-26T20:33:30.264ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
25672024-08-26T20:33:30.264ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 15.536055ms iteration = 32 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
25682024-08-26T20:33:30.982ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 315 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 24c34f0c-69e2-49af-85bf-fca6ad33a564 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
25692024-08-26T20:33:40.249ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 33 reason = Timeout
25702024-08-26T20:33:40.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
25712024-08-26T20:33:40.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
25722024-08-26T20:33:40.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 33 reason = Timeout
25732024-08-26T20:33:40.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
25742024-08-26T20:33:40.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
25752024-08-26T20:33:40.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
25762024-08-26T20:33:40.251ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
25772024-08-26T20:33:40.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
25782024-08-26T20:33:40.252ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 3.109169ms iteration = 33 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
25792024-08-26T20:33:40.253ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
25802024-08-26T20:33:40.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
25812024-08-26T20:33:40.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
25822024-08-26T20:33:40.255ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
25832024-08-26T20:33:40.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
25842024-08-26T20:33:40.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
25852024-08-26T20:33:40.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
25862024-08-26T20:33:40.258ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
25872024-08-26T20:33:40.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
25882024-08-26T20:33:40.263ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
25892024-08-26T20:33:40.263ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 14.93431ms iteration = 33 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
25902024-08-26T20:33:40.981ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 211 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = e4ea4cfd-c5e3-4d72-a063-61a16f00c45e response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
25912024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 34 reason = Timeout
25922024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 12 reason = Timeout
25932024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
25942024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 12 reason = Timeout
25952024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
25962024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
25972024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 12 reason = Timeout
25982024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 12 reason = Timeout
25992024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
26002024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 12 reason = Timeout
26012024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
26022024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 13 reason = Timeout
26032024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 12 reason = Timeout
26042024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
26052024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 12 reason = Timeout
26062024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
26072024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
26082024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
26092024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
26102024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
26112024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 12 reason = Timeout
26122024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 12 reason = Timeout
26132024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
26142024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 12 reason = Timeout
26152024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
26162024-08-26T20:33:50.249ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task started background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:417
26172024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 12 reason = Timeout
26182024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
26192024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 34 reason = Timeout
26202024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 12 reason = Timeout
26212024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
26222024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
26232024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
26242024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
26252024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 12 reason = Timeout
26262024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
26272024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
26282024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
26292024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
26302024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
26312024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
26322024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
26332024-08-26T20:33:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
26342024-08-26T20:33:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
26352024-08-26T20:33:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
26362024-08-26T20:33:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
26372024-08-26T20:33:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
26382024-08-26T20:33:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
26392024-08-26T20:33:50.251ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
26402024-08-26T20:33:50.251ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
26412024-08-26T20:33:50.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
26422024-08-26T20:33:50.251ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
26432024-08-26T20:33:50.251ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 2.432993ms iteration = 12 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
26442024-08-26T20:33:50.251ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
26452024-08-26T20:33:50.252ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
26462024-08-26T20:33:50.252ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
26472024-08-26T20:33:50.252ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1129, tv_nsec: 427045574 } })
26482024-08-26T20:33:50.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
26492024-08-26T20:33:50.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
26502024-08-26T20:33:50.253ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1050
26512024-08-26T20:33:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
26522024-08-26T20:33:50.254ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
26532024-08-26T20:33:50.254ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task done background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:439
26542024-08-26T20:33:50.254ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1129, tv_nsec: 428499807 } })
26552024-08-26T20:33:50.254ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 4.681084ms iteration = 12 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
26562024-08-26T20:33:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
26572024-08-26T20:33:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
26582024-08-26T20:33:50.254ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
26592024-08-26T20:33:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
26602024-08-26T20:33:50.254ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
26612024-08-26T20:33:50.254ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
26622024-08-26T20:33:50.254ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
26632024-08-26T20:33:50.254ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
26642024-08-26T20:33:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
26652024-08-26T20:33:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
26662024-08-26T20:33:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
26672024-08-26T20:33:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
26682024-08-26T20:33:50.255ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
26692024-08-26T20:33:50.255ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
26702024-08-26T20:33:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
26712024-08-26T20:33:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
26722024-08-26T20:33:50.255ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1050
26732024-08-26T20:33:50.255ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
26742024-08-26T20:33:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
26752024-08-26T20:33:50.256ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
26762024-08-26T20:33:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
26772024-08-26T20:33:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
26782024-08-26T20:33:50.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
26792024-08-26T20:33:50.256ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
26802024-08-26T20:33:50.256ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
26812024-08-26T20:33:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
26822024-08-26T20:33:50.256ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 7.002885ms iteration = 12 status = {"errors":[],"garbage_collect_requested":[]}
26832024-08-26T20:33:50.256ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
26842024-08-26T20:33:50.256ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:220 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
26852024-08-26T20:33:50.256ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 7.089246ms iteration = 12 status = {"destroyed_active_vmms":0,"error":null,"saga_start_failures":0,"sagas_completed":0,"sagas_failed":0,"sagas_started":0,"terminated_active_migrations":0}
26862024-08-26T20:33:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
26872024-08-26T20:33:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
26882024-08-26T20:33:50.257ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 7.289428ms iteration = 34 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
26892024-08-26T20:33:50.257ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
26902024-08-26T20:33:50.257ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 8.137596ms iteration = 12 status = {"region_replacement_started_err":0,"region_replacement_started_ok":0}
26912024-08-26T20:33:50.257ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
26922024-08-26T20:33:50.258ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1129, tv_nsec: 432238722 } })
26932024-08-26T20:33:50.258ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 8.413709ms iteration = 12 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
26942024-08-26T20:33:50.258ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
26952024-08-26T20:33:50.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
26962024-08-26T20:33:50.258ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
26972024-08-26T20:33:50.258ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
26982024-08-26T20:33:50.258ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1050 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
26992024-08-26T20:33:50.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
27002024-08-26T20:33:50.259ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27012024-08-26T20:33:50.259ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27022024-08-26T20:33:50.259ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
27032024-08-26T20:33:50.259ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
27042024-08-26T20:33:50.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
27052024-08-26T20:33:50.260ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
27062024-08-26T20:33:50.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
27072024-08-26T20:33:50.261ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:454 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
27082024-08-26T20:33:50.261ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 11.684969ms iteration = 12 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
27092024-08-26T20:33:50.261ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
27102024-08-26T20:33:50.261ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
27112024-08-26T20:33:50.261ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
27122024-08-26T20:33:50.262ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
27132024-08-26T20:33:50.262ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
27142024-08-26T20:33:50.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
27152024-08-26T20:33:50.267ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
27162024-08-26T20:33:50.267ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 18.714146ms iteration = 34 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
27172024-08-26T20:33:50.268ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
27182024-08-26T20:33:50.268ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
27192024-08-26T20:33:50.268ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
27202024-08-26T20:33:50.269ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
27212024-08-26T20:33:50.270ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27222024-08-26T20:33:50.270ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
27232024-08-26T20:33:50.270ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
27242024-08-26T20:33:50.271ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27252024-08-26T20:33:50.271ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
27262024-08-26T20:33:50.271ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
27272024-08-26T20:33:50.272ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
27282024-08-26T20:33:50.273ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27292024-08-26T20:33:50.273ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: c9d3ee27-fe3f-422b-93ce-695a8356d9d2, lookup_type: ById(c9d3ee27-fe3f-422b-93ce-695a8356d9d2) } result = Ok(())
27302024-08-26T20:33:50.273ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
27312024-08-26T20:33:50.274ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
27322024-08-26T20:33:50.274ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
27332024-08-26T20:33:50.274ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
27342024-08-26T20:33:50.274ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
27352024-08-26T20:33:50.274ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 25.477329ms iteration = 12 status = 0
27362024-08-26T20:33:50.287ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 37.695473ms iteration = 13 status = {"physical_disks_added":0}
27372024-08-26T20:33:50.391ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
27382024-08-26T20:33:50.391ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
27392024-08-26T20:33:50.391ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "23867b2b-fa3b-4ae7-be7b-3a984709e649", "content-length": "26", "date": "Mon, 26 Aug 2024 20:33:50 GMT"} })
27402024-08-26T20:33:50.391ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
27412024-08-26T20:33:50.393ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
27422024-08-26T20:33:50.394ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
27432024-08-26T20:33:50.394ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
27442024-08-26T20:33:50.394ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "d06da518-0fe1-4d3e-ae5f-27201c01ab28", "content-length": "26", "date": "Mon, 26 Aug 2024 20:33:50 GMT"} })
27452024-08-26T20:33:50.394ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
27462024-08-26T20:33:50.396ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
27472024-08-26T20:33:50.402ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1058 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 zone_address = fd00:1122:3344:101::2
27482024-08-26T20:33:50.402ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
27492024-08-26T20:33:50.403ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "ea0c0a40-0d4f-4cad-b7a0-0e1744edf4ed", "content-length": "26", "date": "Mon, 26 Aug 2024 20:33:50 GMT"} })
27502024-08-26T20:33:50.403ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1063 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
27512024-08-26T20:33:50.405ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
27522024-08-26T20:33:50.531ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 282.406302ms iteration = 12 status = {}
27532024-08-26T20:33:50.539ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
27542024-08-26T20:33:50.540ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "b9e19985-1bfe-458b-91b0-8d67483e26b2", "content-length": "1", "date": "Mon, 26 Aug 2024 20:33:50 GMT"} })
27552024-08-26T20:33:50.542ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
27562024-08-26T20:33:50.542ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
27572024-08-26T20:33:50.543ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 294.1248ms iteration = 12 status = null
27582024-08-26T20:33:50.561ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
27592024-08-26T20:33:50.561ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "d17c6e5f-fe73-451f-8199-095d8b6dcb78", "content-length": "372", "date": "Mon, 26 Aug 2024 20:33:50 GMT"} })
27602024-08-26T20:33:50.561ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
27612024-08-26T20:33:50.561ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
27622024-08-26T20:33:50.563ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
27632024-08-26T20:33:50.563ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
27642024-08-26T20:33:50.565ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
27652024-08-26T20:33:50.565ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
27662024-08-26T20:33:50.566ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
27672024-08-26T20:33:50.567ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 204, headers: {"x-request-id": "70d2650e-0dd9-4948-9d7a-d99b445dc70a", "date": "Mon, 26 Aug 2024 20:33:50 GMT"} })
27682024-08-26T20:33:50.568ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 318.840362ms iteration = 12 status = {}
27692024-08-26T20:33:50.579ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
27702024-08-26T20:33:50.580ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/v2p", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "c6b965a6-7cd1-4745-82f5-18dce0e7c9a8", "content-length": "691", "date": "Mon, 26 Aug 2024 20:33:50 GMT"} })
27712024-08-26T20:33:50.580ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = sock
27722024-08-26T20:33:50.580ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = sock
27732024-08-26T20:33:50.580ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = sock
27742024-08-26T20:33:50.581ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 332.51579ms iteration = 12 status = {}
27752024-08-26T20:33:50.653ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
27762024-08-26T20:33:50.653ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
27772024-08-26T20:33:50.654ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:147 port = SwitchPort { id: 77948b01-64a9-403e-be7b-4ef5e426dda4, rack_id: 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(735476d6-05ca-4fbd-a182-ab082f700355) } rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
27782024-08-26T20:33:50.654ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
27792024-08-26T20:33:50.655ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
27802024-08-26T20:33:50.660ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
27812024-08-26T20:33:50.661ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/port/qsfp0/settings", query: Some("tag=omicron"), fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "a3816f54-7b28-424c-88c0-333755f511af", "content-length": "121", "date": "Mon, 26 Aug 2024 20:33:50 GMT"} })
27822024-08-26T20:33:50.661ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1714 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.2.174], params: LinkCreate { autoneg: false, fec: None, kr: false, lane: Some(LinkId(0)), speed: Speed40G } }} } switch_port_id = "qsfp0"
27832024-08-26T20:33:50.661ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1741 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = 735476d6-05ca-4fbd-a182-ab082f700355
27842024-08-26T20:33:50.661ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:379 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
27852024-08-26T20:33:50.661ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
27862024-08-26T20:33:50.661ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:392 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
27872024-08-26T20:33:50.661ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:398 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
27882024-08-26T20:33:50.661ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:418 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
27892024-08-26T20:33:50.661ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
27902024-08-26T20:33:50.661ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
27912024-08-26T20:33:50.663ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
27922024-08-26T20:33:50.663ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv4", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "fb929ee4-79e4-40b2-8c74-6d6ea6c17d36", "content-length": "2", "date": "Mon, 26 Aug 2024 20:33:50 GMT"} })
27932024-08-26T20:33:50.663ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
27942024-08-26T20:33:50.663ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv6", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "fcf0a41f-6318-47f5-ab82-f4f5f9ace44b", "content-length": "52", "date": "Mon, 26 Aug 2024 20:33:50 GMT"} })
27952024-08-26T20:33:50.663ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0" }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:467 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
27962024-08-26T20:33:50.663ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
27972024-08-26T20:33:50.755ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/switch-ports", query: None, fragment: None }, status: 204, headers: {"x-request-id": "f241b39f-5c07-4588-9a0a-121c1f636614", "date": "Mon, 26 Aug 2024 20:33:50 GMT"} })
27982024-08-26T20:33:50.755ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:770 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch = Switch0
27992024-08-26T20:33:50.755ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
28002024-08-26T20:33:50.837ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/network-bootstore-config", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "24575d43-3e4d-4f32-8baf-b013e5324f9e", "content-length": "667", "date": "Mon, 26 Aug 2024 20:33:50 GMT"} })
28012024-08-26T20:33:50.837ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
28022024-08-26T20:33:50.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
28032024-08-26T20:33:50.841ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
28042024-08-26T20:33:50.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
28052024-08-26T20:33:50.843ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
28062024-08-26T20:33:50.844ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
28072024-08-26T20:33:50.845ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
28082024-08-26T20:33:50.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
28092024-08-26T20:33:50.852ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 602.843728ms iteration = 12 status = {}
28102024-08-26T20:33:50.982ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 361 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 3b61891e-cd3e-484e-9b53-6fbc5c714246 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
28112024-08-26T20:34:00.249ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 35 reason = Timeout
28122024-08-26T20:34:00.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
28132024-08-26T20:34:00.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
28142024-08-26T20:34:00.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 35 reason = Timeout
28152024-08-26T20:34:00.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
28162024-08-26T20:34:00.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
28172024-08-26T20:34:00.251ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
28182024-08-26T20:34:00.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
28192024-08-26T20:34:00.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
28202024-08-26T20:34:00.252ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 2.666325ms iteration = 35 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
28212024-08-26T20:34:00.253ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
28222024-08-26T20:34:00.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
28232024-08-26T20:34:00.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
28242024-08-26T20:34:00.255ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
28252024-08-26T20:34:00.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
28262024-08-26T20:34:00.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
28272024-08-26T20:34:00.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
28282024-08-26T20:34:00.258ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
28292024-08-26T20:34:00.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
28302024-08-26T20:34:00.264ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
28312024-08-26T20:34:00.264ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 15.897799ms iteration = 35 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
28322024-08-26T20:34:00.982ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 218 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = fa3f4550-af58-4d62-99b8-7fcdde61d41f response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
28332024-08-26T20:34:10.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 36 reason = Timeout
28342024-08-26T20:34:10.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
28352024-08-26T20:34:10.249ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 36 reason = Timeout
28362024-08-26T20:34:10.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
28372024-08-26T20:34:10.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
28382024-08-26T20:34:10.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
28392024-08-26T20:34:10.251ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
28402024-08-26T20:34:10.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
28412024-08-26T20:34:10.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
28422024-08-26T20:34:10.252ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 2.861767ms iteration = 36 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
28432024-08-26T20:34:10.253ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
28442024-08-26T20:34:10.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
28452024-08-26T20:34:10.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
28462024-08-26T20:34:10.255ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
28472024-08-26T20:34:10.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
28482024-08-26T20:34:10.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
28492024-08-26T20:34:10.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
28502024-08-26T20:34:10.258ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
28512024-08-26T20:34:10.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
28522024-08-26T20:34:10.264ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
28532024-08-26T20:34:10.264ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 15.268882ms iteration = 36 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
28542024-08-26T20:34:10.982ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 213 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 624475f9-45a8-4969-8b49-aa1ce2f92249 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
28552024-08-26T20:34:20.248ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 9 reason = Timeout
28562024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 7 reason = Timeout
28572024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 7 reason = Timeout
28582024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 13 reason = Timeout
28592024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
28602024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
28612024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 7 reason = Timeout
28622024-08-26T20:34:20.249ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-08-26 20:24:20.249011529 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
28632024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 7 reason = Timeout
28642024-08-26T20:34:20.249ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
28652024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
28662024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = crdb_node_id_collector iteration = 4 reason = Timeout
28672024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
28682024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 13 reason = Timeout
28692024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
28702024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
28712024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 13 reason = Timeout
28722024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
28732024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 13 reason = Timeout
28742024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 13 reason = Timeout
28752024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
28762024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 14 reason = Timeout
28772024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 37 reason = Timeout
28782024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
28792024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 8 reason = Timeout
28802024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
28812024-08-26T20:34:20.249ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:80 reason = blueprint disabled target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460
28822024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 13.26\u{b5}s iteration = 8 status = {"error":"blueprint disabled","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460"}
28832024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 7 reason = Timeout
28842024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 9 reason = Timeout
28852024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 7 reason = Timeout
28862024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
28872024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
28882024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 7 reason = Timeout
28892024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
28902024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 13 reason = Timeout
28912024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
28922024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
28932024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 13 reason = Timeout
28942024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
28952024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 13 reason = Timeout
28962024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 37 reason = Timeout
28972024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
28982024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
28992024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 13 reason = Timeout
29002024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
29012024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 13 reason = Timeout
29022024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
29032024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
29042024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
29052024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 13 reason = Timeout
29062024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
29072024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 7 reason = Timeout
29082024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 13 reason = Timeout
29092024-08-26T20:34:20.249ZINFOnexus (BackgroundTasks): Abandoned VMM reaper running background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:76
29102024-08-26T20:34:20.249ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task started background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:417
29112024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
29122024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
29132024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 7 reason = Timeout
29142024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
29152024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 13 reason = Timeout
29162024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
29172024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
29182024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
29192024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
29202024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
29212024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
29222024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
29232024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
29242024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
29252024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
29262024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
29272024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
29282024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
29292024-08-26T20:34:20.249ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("be00167b-db1f-4cc1-86f1-a2af7f1c31f8.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("d2b5c33b-9149-4357-86b1-60e922261ec4.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("d2b5c33b-9149-4357-86b1-60e922261ec4.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 1159, tv_nsec: 423924084 } })
29302024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
29312024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
29322024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
29332024-08-26T20:34:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
29342024-08-26T20:34:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
29352024-08-26T20:34:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
29362024-08-26T20:34:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
29372024-08-26T20:34:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
29382024-08-26T20:34:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
29392024-08-26T20:34:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
29402024-08-26T20:34:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
29412024-08-26T20:34:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
29422024-08-26T20:34:20.250ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
29432024-08-26T20:34:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
29442024-08-26T20:34:20.250ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
29452024-08-26T20:34:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
29462024-08-26T20:34:20.250ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
29472024-08-26T20:34:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
29482024-08-26T20:34:20.250ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
29492024-08-26T20:34:20.250ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("316f64f2-8fa5-43f3-8d5d-92212e3046c4.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("35cb7d67-b9f1-4dcf-bac3-28244fb25d88.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("e37633b8-b58b-40c2-b3c2-6873358565b7.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("e37633b8-b58b-40c2-b3c2-6873358565b7.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:3::1))) }], valid_until: Instant { tv_sec: 1159, tv_nsec: 424977304 } })
29502024-08-26T20:34:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
29512024-08-26T20:34:20.251ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
29522024-08-26T20:34:20.251ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 2.505733ms iteration = 7 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
29532024-08-26T20:34:20.252ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
29542024-08-26T20:34:20.252ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
29552024-08-26T20:34:20.252ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
29562024-08-26T20:34:20.252ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.537164ms iteration = 13 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
29572024-08-26T20:34:20.252ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
29582024-08-26T20:34:20.253ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
29592024-08-26T20:34:20.253ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } dns_group = internal
29602024-08-26T20:34:20.253ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 4.3322ms iteration = 7 status = {"addresses":["[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353","[fd00:1122:3344:3::1]:5353"]}
29612024-08-26T20:34:20.253ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1159, tv_nsec: 427720640 } })
29622024-08-26T20:34:20.253ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1159, tv_nsec: 427876352 } })
29632024-08-26T20:34:20.253ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
29642024-08-26T20:34:20.254ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-08-26 20:24:20.249011529 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
29652024-08-26T20:34:20.254ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 5.188208ms iteration = 7 status = {"expiration":"2024-08-26T20:24:20.249011529Z","pruned":[]}
29662024-08-26T20:34:20.254ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1050
29672024-08-26T20:34:20.254ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1050
29682024-08-26T20:34:20.255ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 5.553482ms iteration = 7 status = {"errors":[],"found_port_ok":[]}
29692024-08-26T20:34:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
29702024-08-26T20:34:20.255ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
29712024-08-26T20:34:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
29722024-08-26T20:34:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
29732024-08-26T20:34:20.255ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
29742024-08-26T20:34:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
29752024-08-26T20:34:20.256ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0
29762024-08-26T20:34:20.256ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:189 found = 0 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0
29772024-08-26T20:34:20.256ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 6.556761ms iteration = 7 status = {"error":null,"error_count":0,"found":0,"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0}
29782024-08-26T20:34:20.256ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
29792024-08-26T20:34:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
29802024-08-26T20:34:20.256ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
29812024-08-26T20:34:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
29822024-08-26T20:34:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
29832024-08-26T20:34:20.256ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
29842024-08-26T20:34:20.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
29852024-08-26T20:34:20.256ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
29862024-08-26T20:34:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
29872024-08-26T20:34:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
29882024-08-26T20:34:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
29892024-08-26T20:34:20.257ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
29902024-08-26T20:34:20.257ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
29912024-08-26T20:34:20.257ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
29922024-08-26T20:34:20.257ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
29932024-08-26T20:34:20.257ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
29942024-08-26T20:34:20.257ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
29952024-08-26T20:34:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
29962024-08-26T20:34:20.257ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
29972024-08-26T20:34:20.257ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
29982024-08-26T20:34:20.257ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 8.343909ms iteration = 13 status = {"errors":[],"garbage_collect_requested":[]}
29992024-08-26T20:34:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
30002024-08-26T20:34:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
30012024-08-26T20:34:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
30022024-08-26T20:34:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
30032024-08-26T20:34:20.258ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
30042024-08-26T20:34:20.258ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
30052024-08-26T20:34:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
30062024-08-26T20:34:20.259ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1159, tv_nsec: 433318152 } })
30072024-08-26T20:34:20.259ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
30082024-08-26T20:34:20.259ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30092024-08-26T20:34:20.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
30102024-08-26T20:34:20.259ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1050 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
30112024-08-26T20:34:20.259ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
30122024-08-26T20:34:20.259ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:220 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
30132024-08-26T20:34:20.259ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 10.056654ms iteration = 13 status = {"destroyed_active_vmms":0,"error":null,"saga_start_failures":0,"sagas_completed":0,"sagas_failed":0,"sagas_started":0,"terminated_active_migrations":0}
30142024-08-26T20:34:20.259ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30152024-08-26T20:34:20.259ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
30162024-08-26T20:34:20.259ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
30172024-08-26T20:34:20.259ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
30182024-08-26T20:34:20.259ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 10.769511ms iteration = 7 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
30192024-08-26T20:34:20.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
30202024-08-26T20:34:20.259ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30212024-08-26T20:34:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
30222024-08-26T20:34:20.260ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
30232024-08-26T20:34:20.260ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 11.007583ms iteration = 13 status = {"region_replacement_started_err":0,"region_replacement_started_ok":0}
30242024-08-26T20:34:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
30252024-08-26T20:34:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
30262024-08-26T20:34:20.262ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
30272024-08-26T20:34:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
30282024-08-26T20:34:20.263ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
30292024-08-26T20:34:20.263ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30302024-08-26T20:34:20.264ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 14.778259ms iteration = 37 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
30312024-08-26T20:34:20.264ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
30322024-08-26T20:34:20.265ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
30332024-08-26T20:34:20.265ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
30342024-08-26T20:34:20.266ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
30352024-08-26T20:34:20.266ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30362024-08-26T20:34:20.267ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
30372024-08-26T20:34:20.267ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
30382024-08-26T20:34:20.267ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
30392024-08-26T20:34:20.267ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
30402024-08-26T20:34:20.267ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
30412024-08-26T20:34:20.267ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
30422024-08-26T20:34:20.267ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
30432024-08-26T20:34:20.267ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
30442024-08-26T20:34:20.267ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
30452024-08-26T20:34:20.267ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
30462024-08-26T20:34:20.267ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
30472024-08-26T20:34:20.267ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
30482024-08-26T20:34:20.268ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
30492024-08-26T20:34:20.268ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
30502024-08-26T20:34:20.268ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
30512024-08-26T20:34:20.268ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
30522024-08-26T20:34:20.268ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
30532024-08-26T20:34:20.268ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
30542024-08-26T20:34:20.268ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
30552024-08-26T20:34:20.273ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
30562024-08-26T20:34:20.273ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
30572024-08-26T20:34:20.273ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
30582024-08-26T20:34:20.274ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
30592024-08-26T20:34:20.274ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30602024-08-26T20:34:20.275ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 25.600389ms iteration = 13 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
30612024-08-26T20:34:20.275ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30622024-08-26T20:34:20.275ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
30632024-08-26T20:34:20.275ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
30642024-08-26T20:34:20.276ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task done background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:439
30652024-08-26T20:34:20.276ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 27.331986ms iteration = 13 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
30662024-08-26T20:34:20.276ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30672024-08-26T20:34:20.277ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
30682024-08-26T20:34:20.277ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
30692024-08-26T20:34:20.277ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:454 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
30702024-08-26T20:34:20.277ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 27.688889ms iteration = 13 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
30712024-08-26T20:34:20.277ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
30722024-08-26T20:34:20.277ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
30732024-08-26T20:34:20.277ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
30742024-08-26T20:34:20.277ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30752024-08-26T20:34:20.278ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
30762024-08-26T20:34:20.278ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
30772024-08-26T20:34:20.278ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
30782024-08-26T20:34:20.278ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30792024-08-26T20:34:20.278ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30802024-08-26T20:34:20.279ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
30812024-08-26T20:34:20.279ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = b82a4171-23b2-444b-9597-09085a7a2fb8 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
30822024-08-26T20:34:20.279ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
30832024-08-26T20:34:20.279ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
30842024-08-26T20:34:20.279ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
30852024-08-26T20:34:20.279ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = d840a78c-2f0d-4cdf-9962-31a3dae384a2 dns_zone_name = oxide.test found_so_far = 0 version = 2
30862024-08-26T20:34:20.279ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
30872024-08-26T20:34:20.280ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
30882024-08-26T20:34:20.280ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
30892024-08-26T20:34:20.280ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
30902024-08-26T20:34:20.280ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
30912024-08-26T20:34:20.280ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
30922024-08-26T20:34:20.280ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
30932024-08-26T20:34:20.280ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
30942024-08-26T20:34:20.280ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30952024-08-26T20:34:20.281ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30962024-08-26T20:34:20.281ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
30972024-08-26T20:34:20.281ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
30982024-08-26T20:34:20.281ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
30992024-08-26T20:34:20.282ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
31002024-08-26T20:34:20.282ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
31012024-08-26T20:34:20.282ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
31022024-08-26T20:34:20.282ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
31032024-08-26T20:34:20.283ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
31042024-08-26T20:34:20.284ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
31052024-08-26T20:34:20.284ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
31062024-08-26T20:34:20.284ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
31072024-08-26T20:34:20.284ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
31082024-08-26T20:34:20.284ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 35.708574ms iteration = 14 status = {"physical_disks_added":0}
31092024-08-26T20:34:20.284ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
31102024-08-26T20:34:20.285ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
31112024-08-26T20:34:20.285ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
31122024-08-26T20:34:20.285ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
31132024-08-26T20:34:20.285ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
31142024-08-26T20:34:20.285ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
31152024-08-26T20:34:20.285ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
31162024-08-26T20:34:20.286ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
31172024-08-26T20:34:20.286ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
31182024-08-26T20:34:20.286ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = b82a4171-23b2-444b-9597-09085a7a2fb8 dns_zone_name = control-plane.oxide.internal found_so_far = 50 version = 1
31192024-08-26T20:34:20.286ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
31202024-08-26T20:34:20.286ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-08-26 20:29:01.089496 UTC dns_group = internal generation = 1
31212024-08-26T20:34:20.286ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 37.43955ms iteration = 7 status = {"generation":1}
31222024-08-26T20:34:20.287ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = d840a78c-2f0d-4cdf-9962-31a3dae384a2 dns_zone_name = oxide.test found_so_far = 1 version = 2
31232024-08-26T20:34:20.287ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
31242024-08-26T20:34:20.287ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-08-26 20:29:01.487349 UTC dns_group = external generation = 2
31252024-08-26T20:34:20.287ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 37.869855ms iteration = 7 status = {"generation":2}
31262024-08-26T20:34:20.288ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: c9d3ee27-fe3f-422b-93ce-695a8356d9d2, lookup_type: ById(c9d3ee27-fe3f-422b-93ce-695a8356d9d2) } result = Ok(())
31272024-08-26T20:34:20.288ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
31282024-08-26T20:34:20.288ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
31292024-08-26T20:34:20.289ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 40.037495ms iteration = 13 status = 0
31302024-08-26T20:34:20.289ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
31312024-08-26T20:34:20.289ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 40.237696ms iteration = 37 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
31322024-08-26T20:34:20.298ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
31332024-08-26T20:34:20.298ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
31342024-08-26T20:34:20.299ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
31352024-08-26T20:34:20.300ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
31362024-08-26T20:34:20.300ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
31372024-08-26T20:34:20.300ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
31382024-08-26T20:34:20.310ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-08-26T20:28:18.089116Z, time_modified: 2024-08-26T20:28:18.089116Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 80a72585-f38a-4ccb-b69a-caafab415ca2, db_silo: Silo { identity: SiloIdentity { id: 80a72585-f38a-4ccb-b69a-caafab415ca2, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-08-26T20:29:01.450089Z, time_modified: 2024-08-26T20:29:01.450089Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 80a72585-f38a-4ccb-b69a-caafab415ca2, db_silo: Silo { identity: SiloIdentity { id: 80a72585-f38a-4ccb-b69a-caafab415ca2, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-08-26T20:29:01.450089Z, time_modified: 2024-08-26T20:29:01.450089Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923" }] }) }
31392024-08-26T20:34:20.310ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 62.00309ms iteration = 7 status = {"by_dns_name":{"default-silo.sys.oxide.test":{"silo_id":"001de000-5110-4000-8000-000000000000","tls_certs":[]},"recovery.sys.oxide.test":{"silo_id":"80a72585-f38a-4ccb-b69a-caafab415ca2","tls_certs":["81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923"]}},"default_endpoint":{"silo_id":"80a72585-f38a-4ccb-b69a-caafab415ca2","tls_certs":["81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923"]},"warnings":["silo 001de000-5110-4000-8000-000000000000 with DNS name \\"default-silo.sys.oxide.test\\" has no usable certificates"]}
31402024-08-26T20:34:20.441ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
31412024-08-26T20:34:20.448ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
31422024-08-26T20:34:20.448ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
31432024-08-26T20:34:20.448ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "ab9fc808-96b6-4191-818f-35efa8b92285", "content-length": "26", "date": "Mon, 26 Aug 2024 20:34:20 GMT"} })
31442024-08-26T20:34:20.448ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
31452024-08-26T20:34:20.450ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
31462024-08-26T20:34:20.454ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
31472024-08-26T20:34:20.455ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "57a1d29f-28af-4a16-b3d1-b3bc0eedb86d", "content-length": "372", "date": "Mon, 26 Aug 2024 20:34:19 GMT"} })
31482024-08-26T20:34:20.455ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
31492024-08-26T20:34:20.455ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
31502024-08-26T20:34:20.456ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
31512024-08-26T20:34:20.456ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
31522024-08-26T20:34:20.457ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
31532024-08-26T20:34:20.457ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
31542024-08-26T20:34:20.457ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
31552024-08-26T20:34:20.457ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
31562024-08-26T20:34:20.457ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
31572024-08-26T20:34:20.459ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
31582024-08-26T20:34:20.459ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
31592024-08-26T20:34:20.459ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
31602024-08-26T20:34:20.459ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
31612024-08-26T20:34:20.459ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
31622024-08-26T20:34:20.459ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
31632024-08-26T20:34:20.460ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
31642024-08-26T20:34:20.460ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
31652024-08-26T20:34:20.460ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
31662024-08-26T20:34:20.460ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
31672024-08-26T20:34:20.461ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 204, headers: {"x-request-id": "e3408cb9-c2db-4bf7-9e8a-c0e649942a01", "date": "Mon, 26 Aug 2024 20:34:19 GMT"} })
31682024-08-26T20:34:20.462ZDEBGnexus (BackgroundTasks): activation complete background_task = crdb_node_id_collector elapsed = 213.062203ms iteration = 4 status = {"nsuccess":5}
31692024-08-26T20:34:20.462ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 213.186874ms iteration = 13 status = {}
31702024-08-26T20:34:20.560ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
31712024-08-26T20:34:20.592ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
31722024-08-26T20:34:20.592ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
31732024-08-26T20:34:20.592ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "8c85b675-df26-409a-a4ed-9cfff2c47202", "content-length": "26", "date": "Mon, 26 Aug 2024 20:34:20 GMT"} })
31742024-08-26T20:34:20.592ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
31752024-08-26T20:34:20.594ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
31762024-08-26T20:34:20.600ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1058 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 zone_address = fd00:1122:3344:101::2
31772024-08-26T20:34:20.600ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
31782024-08-26T20:34:20.601ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "e1a27ecc-a0d4-4064-ae8e-b587553720fd", "content-length": "26", "date": "Mon, 26 Aug 2024 20:34:20 GMT"} })
31792024-08-26T20:34:20.601ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1063 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
31802024-08-26T20:34:20.604ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
31812024-08-26T20:34:20.604ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
31822024-08-26T20:34:20.605ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/v2p", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "b51eb531-9ad0-4163-a25d-d4c0273b8b98", "content-length": "691", "date": "Mon, 26 Aug 2024 20:34:19 GMT"} })
31832024-08-26T20:34:20.605ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = sock
31842024-08-26T20:34:20.605ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = sock
31852024-08-26T20:34:20.605ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = sock
31862024-08-26T20:34:20.608ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 358.899407ms iteration = 13 status = {}
31872024-08-26T20:34:20.616ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 367.607717ms iteration = 13 status = {}
31882024-08-26T20:34:20.628ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
31892024-08-26T20:34:20.629ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::9)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "2126e561-59fd-4d96-a60d-d7c795ed50ef", "date": "Mon, 26 Aug 2024 20:34:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
31902024-08-26T20:34:20.629ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
31912024-08-26T20:34:20.630ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::8)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "370bdb27-7e6d-44f1-bf68-0f83f2365779", "date": "Mon, 26 Aug 2024 20:34:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
31922024-08-26T20:34:20.630ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
31932024-08-26T20:34:20.632ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
31942024-08-26T20:34:20.632ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 383.973411ms iteration = 9 status = {"generation":2,"server_results":{"[fd00:1122:3344:101::8]:5353":{"Ok":null},"[fd00:1122:3344:101::9]:5353":{"Ok":null}}}
31952024-08-26T20:34:20.680ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
31962024-08-26T20:34:20.683ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
31972024-08-26T20:34:20.684ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "550fab60-0ab4-4a47-8109-c9e3d8cb4563", "content-length": "1", "date": "Mon, 26 Aug 2024 20:34:20 GMT"} })
31982024-08-26T20:34:20.684ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
31992024-08-26T20:34:20.685ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
32002024-08-26T20:34:20.686ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 437.32426ms iteration = 13 status = null
32012024-08-26T20:34:20.742ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
32022024-08-26T20:34:20.743ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:3::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "5ad06a8f-18d2-444e-8ff4-50c44c737fc1", "date": "Mon, 26 Aug 2024 20:34:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
32032024-08-26T20:34:20.743ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
32042024-08-26T20:34:20.745ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:1::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "cbccb553-22d2-4e7a-a48d-af59f9f6fb8f", "date": "Mon, 26 Aug 2024 20:34:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
32052024-08-26T20:34:20.745ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
32062024-08-26T20:34:20.747ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:2::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "98350135-3793-44a5-ab74-976569276f3d", "date": "Mon, 26 Aug 2024 20:34:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
32072024-08-26T20:34:20.747ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
32082024-08-26T20:34:20.749ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
32092024-08-26T20:34:20.749ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 500.38982ms iteration = 9 status = {"generation":1,"server_results":{"[fd00:1122:3344:1::1]:5353":{"Ok":null},"[fd00:1122:3344:2::1]:5353":{"Ok":null},"[fd00:1122:3344:3::1]:5353":{"Ok":null}}}
32102024-08-26T20:34:20.779ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
32112024-08-26T20:34:20.780ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
32122024-08-26T20:34:20.781ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:147 port = SwitchPort { id: 77948b01-64a9-403e-be7b-4ef5e426dda4, rack_id: 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(735476d6-05ca-4fbd-a182-ab082f700355) } rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
32132024-08-26T20:34:20.781ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
32142024-08-26T20:34:20.781ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
32152024-08-26T20:34:20.791ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
32162024-08-26T20:34:20.791ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/port/qsfp0/settings", query: Some("tag=omicron"), fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "a146acf3-e7f8-46b8-84b2-9900c31f37c1", "content-length": "121", "date": "Mon, 26 Aug 2024 20:34:20 GMT"} })
32172024-08-26T20:34:20.791ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1714 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.2.174], params: LinkCreate { autoneg: false, fec: None, kr: false, lane: Some(LinkId(0)), speed: Speed40G } }} } switch_port_id = "qsfp0"
32182024-08-26T20:34:20.791ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1741 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = 735476d6-05ca-4fbd-a182-ab082f700355
32192024-08-26T20:34:20.791ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:379 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
32202024-08-26T20:34:20.791ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
32212024-08-26T20:34:20.791ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:392 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
32222024-08-26T20:34:20.791ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:398 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
32232024-08-26T20:34:20.791ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:418 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
32242024-08-26T20:34:20.791ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
32252024-08-26T20:34:20.791ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
32262024-08-26T20:34:20.793ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
32272024-08-26T20:34:20.793ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv4", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "9cdb3aad-4600-4542-9a5f-16afb1614389", "content-length": "2", "date": "Mon, 26 Aug 2024 20:34:20 GMT"} })
32282024-08-26T20:34:20.793ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
32292024-08-26T20:34:20.794ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv6", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "2abc6938-32fd-4ffc-ad03-8c501db60c67", "content-length": "52", "date": "Mon, 26 Aug 2024 20:34:20 GMT"} })
32302024-08-26T20:34:20.794ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0" }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:467 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
32312024-08-26T20:34:20.794ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
32322024-08-26T20:34:20.880ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/switch-ports", query: None, fragment: None }, status: 204, headers: {"x-request-id": "93b25249-d040-49e1-978c-9fcfa2ce1651", "date": "Mon, 26 Aug 2024 20:34:20 GMT"} })
32332024-08-26T20:34:20.880ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:770 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch = Switch0
32342024-08-26T20:34:20.880ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
32352024-08-26T20:34:20.881ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/network-bootstore-config", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "16859d32-9448-4988-ab2d-62d4ab1b6540", "content-length": "667", "date": "Mon, 26 Aug 2024 20:34:20 GMT"} })
32362024-08-26T20:34:20.881ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
32372024-08-26T20:34:20.881ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
32382024-08-26T20:34:20.884ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
32392024-08-26T20:34:20.884ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
32402024-08-26T20:34:20.886ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
32412024-08-26T20:34:20.887ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
32422024-08-26T20:34:20.888ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
32432024-08-26T20:34:20.888ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
32442024-08-26T20:34:20.896ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 647.071143ms iteration = 13 status = {}
32452024-08-26T20:34:20.982ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 268 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = ea5cca98-dca0-4e7a-bd49-6de13867e395 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
32462024-08-26T20:34:20.984ZDEBGnexus (producer-registration-task): registering / renewing oximeter producer lease with Nexus
32472024-08-26T20:34:20.984ZDEBGnexus (producer-registration-task): using nexus address for registration addr = [fd00:1122:3344:101::c]:12221
32482024-08-26T20:34:21.015ZDEBGnexus (producer-registration-task): client request body = Some(Body) method = POST uri = http://[fd00:1122:3344:101::c]:12221/metrics/producers
32492024-08-26T20:34:21.015ZINFOnexus (dropshot_internal): accepted connection file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775 local_addr = [fd00:1122:3344:101::c]:12221 remote_addr = [fd00:1122:3344:101::c]:43080
32502024-08-26T20:34:21.015ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:43080 req_id = 011c08c3-8391-47b3-8975-bfd5aed7b1bf roles = RoleSet { roles: {} } uri = /metrics/producers
32512024-08-26T20:34:21.015ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:43080 req_id = 011c08c3-8391-47b3-8975-bfd5aed7b1bf resource = Database result = Ok(()) uri = /metrics/producers
32522024-08-26T20:34:21.047ZINFOnexus: registered oximeter collector client file = nexus/src/app/oximeter.rs:338 id = 66888b78-0269-44c1-8d42-ab8ec11561af
32532024-08-26T20:34:21.047ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:43080 req_id = 011c08c3-8391-47b3-8975-bfd5aed7b1bf roles = RoleSet { roles: {} } uri = /metrics/producers
32542024-08-26T20:34:21.047ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:43080 req_id = 011c08c3-8391-47b3-8975-bfd5aed7b1bf resource = Database result = Ok(()) uri = /metrics/producers
32552024-08-26T20:34:21.093ZDEBGnexus: client request body = Some(Body) method = POST oximeter-collector = 66888b78-0269-44c1-8d42-ab8ec11561af uri = http://[fd00:1122:3344:101::d]:12223/producers
32562024-08-26T20:34:21.093ZDEBGnexus: client response oximeter-collector = 66888b78-0269-44c1-8d42-ab8ec11561af result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::d)), port: Some(12223), path: "/producers", query: None, fragment: None }, status: 204, headers: {"x-request-id": "3a904847-d8a2-4e53-b3b9-7e0edc3eaaaf", "date": "Mon, 26 Aug 2024 20:34:21 GMT"} })
32572024-08-26T20:34:21.093ZINFOnexus: assigned collector to new producer collector_id = 66888b78-0269-44c1-8d42-ab8ec11561af file = nexus/src/app/oximeter.rs:125 producer_id = ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
32582024-08-26T20:34:21.095ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 79844 local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:43080 req_id = 011c08c3-8391-47b3-8975-bfd5aed7b1bf response_code = 201 uri = /metrics/producers
32592024-08-26T20:34:21.095ZDEBGnexus (producer-registration-task): client response result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, status: 201, headers: {"content-type": "application/json", "x-request-id": "011c08c3-8391-47b3-8975-bfd5aed7b1bf", "content-length": "41", "date": "Mon, 26 Aug 2024 20:34:21 GMT"} })
32602024-08-26T20:34:21.096ZDEBGnexus (producer-registration-task): registered with nexus successfully
32612024-08-26T20:34:21.096ZDEBGnexus (producer-registration-task): pausing until time to renew lease lease_duration = 600s wait_period = 150s
32622024-08-26T20:34:30.248ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 38 reason = Timeout
32632024-08-26T20:34:30.248ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
32642024-08-26T20:34:30.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
32652024-08-26T20:34:30.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 38 reason = Timeout
32662024-08-26T20:34:30.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
32672024-08-26T20:34:30.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
32682024-08-26T20:34:30.250ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
32692024-08-26T20:34:30.250ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
32702024-08-26T20:34:30.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
32712024-08-26T20:34:30.252ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 2.781936ms iteration = 38 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
32722024-08-26T20:34:30.252ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
32732024-08-26T20:34:30.252ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
32742024-08-26T20:34:30.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
32752024-08-26T20:34:30.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
32762024-08-26T20:34:30.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
32772024-08-26T20:34:30.255ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
32782024-08-26T20:34:30.257ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
32792024-08-26T20:34:30.257ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
32802024-08-26T20:34:30.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
32812024-08-26T20:34:30.264ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
32822024-08-26T20:34:30.264ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 15.749517ms iteration = 38 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
32832024-08-26T20:34:31.095ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 306 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 2b84626f-1b1d-475d-9484-b789b3e4564e response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
32842024-08-26T20:34:40.248ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 39 reason = Timeout
32852024-08-26T20:34:40.248ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
32862024-08-26T20:34:40.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
32872024-08-26T20:34:40.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 39 reason = Timeout
32882024-08-26T20:34:40.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
32892024-08-26T20:34:40.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
32902024-08-26T20:34:40.250ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
32912024-08-26T20:34:40.251ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
32922024-08-26T20:34:40.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
32932024-08-26T20:34:40.252ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 2.843826ms iteration = 39 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
32942024-08-26T20:34:40.253ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
32952024-08-26T20:34:40.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
32962024-08-26T20:34:40.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
32972024-08-26T20:34:40.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
32982024-08-26T20:34:40.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
32992024-08-26T20:34:40.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
33002024-08-26T20:34:40.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
33012024-08-26T20:34:40.258ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
33022024-08-26T20:34:40.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
33032024-08-26T20:34:40.264ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
33042024-08-26T20:34:40.264ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 15.857588ms iteration = 39 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
33052024-08-26T20:34:41.096ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 857 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = bc134107-4782-4e37-9433-286c1d226d0b response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
33062024-08-26T20:34:50.248ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 40 reason = Timeout
33072024-08-26T20:34:50.248ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 14 reason = Timeout
33082024-08-26T20:34:50.248ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
33092024-08-26T20:34:50.248ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
33102024-08-26T20:34:50.248ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 14 reason = Timeout
33112024-08-26T20:34:50.248ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
33122024-08-26T20:34:50.248ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 14 reason = Timeout
33132024-08-26T20:34:50.248ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 14 reason = Timeout
33142024-08-26T20:34:50.248ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
33152024-08-26T20:34:50.248ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 15 reason = Timeout
33162024-08-26T20:34:50.248ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 14 reason = Timeout
33172024-08-26T20:34:50.248ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
33182024-08-26T20:34:50.248ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
33192024-08-26T20:34:50.248ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 14 reason = Timeout
33202024-08-26T20:34:50.248ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
33212024-08-26T20:34:50.248ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 14 reason = Timeout
33222024-08-26T20:34:50.248ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
33232024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
33242024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
33252024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
33262024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
33272024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
33282024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
33292024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
33302024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
33312024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 14 reason = Timeout
33322024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 14 reason = Timeout
33332024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
33342024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 14 reason = Timeout
33352024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
33362024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
33372024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
33382024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 40 reason = Timeout
33392024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
33402024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 14 reason = Timeout
33412024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
33422024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 14 reason = Timeout
33432024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
33442024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 14 reason = Timeout
33452024-08-26T20:34:50.249ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task started background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:417
33462024-08-26T20:34:50.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
33472024-08-26T20:34:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
33482024-08-26T20:34:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
33492024-08-26T20:34:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
33502024-08-26T20:34:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
33512024-08-26T20:34:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
33522024-08-26T20:34:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
33532024-08-26T20:34:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
33542024-08-26T20:34:50.251ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
33552024-08-26T20:34:50.251ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
33562024-08-26T20:34:50.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
33572024-08-26T20:34:50.251ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
33582024-08-26T20:34:50.251ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.045129ms iteration = 14 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
33592024-08-26T20:34:50.252ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1189, tv_nsec: 426443638 } })
33602024-08-26T20:34:50.252ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
33612024-08-26T20:34:50.253ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1189, tv_nsec: 427326346 } })
33622024-08-26T20:34:50.253ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1050
33632024-08-26T20:34:50.253ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 3.510503ms iteration = 14 status = {"errors":[],"garbage_collect_requested":[]}
33642024-08-26T20:34:50.253ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
33652024-08-26T20:34:50.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
33662024-08-26T20:34:50.253ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1050
33672024-08-26T20:34:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
33682024-08-26T20:34:50.254ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
33692024-08-26T20:34:50.254ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 5.952366ms iteration = 14 status = {"region_replacement_started_err":0,"region_replacement_started_ok":0}
33702024-08-26T20:34:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
33712024-08-26T20:34:50.254ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
33722024-08-26T20:34:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
33732024-08-26T20:34:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
33742024-08-26T20:34:50.254ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
33752024-08-26T20:34:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
33762024-08-26T20:34:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
33772024-08-26T20:34:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
33782024-08-26T20:34:50.255ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
33792024-08-26T20:34:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
33802024-08-26T20:34:50.255ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
33812024-08-26T20:34:50.255ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
33822024-08-26T20:34:50.255ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
33832024-08-26T20:34:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
33842024-08-26T20:34:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
33852024-08-26T20:34:50.256ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
33862024-08-26T20:34:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
33872024-08-26T20:34:50.256ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
33882024-08-26T20:34:50.256ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 7.149967ms iteration = 40 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
33892024-08-26T20:34:50.257ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1189, tv_nsec: 431274783 } })
33902024-08-26T20:34:50.257ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
33912024-08-26T20:34:50.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
33922024-08-26T20:34:50.257ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
33932024-08-26T20:34:50.257ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
33942024-08-26T20:34:50.257ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
33952024-08-26T20:34:50.257ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
33962024-08-26T20:34:50.257ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
33972024-08-26T20:34:50.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
33982024-08-26T20:34:50.257ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
33992024-08-26T20:34:50.257ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1050 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
34002024-08-26T20:34:50.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
34012024-08-26T20:34:50.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
34022024-08-26T20:34:50.258ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
34032024-08-26T20:34:50.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
34042024-08-26T20:34:50.258ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
34052024-08-26T20:34:50.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
34062024-08-26T20:34:50.259ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
34072024-08-26T20:34:50.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
34082024-08-26T20:34:50.259ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
34092024-08-26T20:34:50.259ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:220 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
34102024-08-26T20:34:50.259ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 9.806971ms iteration = 14 status = {"destroyed_active_vmms":0,"error":null,"saga_start_failures":0,"sagas_completed":0,"sagas_failed":0,"sagas_started":0,"terminated_active_migrations":0}
34112024-08-26T20:34:50.259ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
34122024-08-26T20:34:50.259ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
34132024-08-26T20:34:50.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
34142024-08-26T20:34:50.260ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task done background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:439
34152024-08-26T20:34:50.260ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 10.862782ms iteration = 14 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
34162024-08-26T20:34:50.260ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
34172024-08-26T20:34:50.260ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
34182024-08-26T20:34:50.260ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
34192024-08-26T20:34:50.260ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
34202024-08-26T20:34:50.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
34212024-08-26T20:34:50.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
34222024-08-26T20:34:50.262ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
34232024-08-26T20:34:50.262ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
34242024-08-26T20:34:50.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
34252024-08-26T20:34:50.264ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 16.259722ms iteration = 15 status = {"physical_disks_added":0}
34262024-08-26T20:34:50.266ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
34272024-08-26T20:34:50.266ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 18.09979ms iteration = 40 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
34282024-08-26T20:34:50.267ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
34292024-08-26T20:34:50.267ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
34302024-08-26T20:34:50.268ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
34312024-08-26T20:34:50.270ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
34322024-08-26T20:34:50.270ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
34332024-08-26T20:34:50.270ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
34342024-08-26T20:34:50.271ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
34352024-08-26T20:34:50.272ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
34362024-08-26T20:34:50.272ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
34372024-08-26T20:34:50.272ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
34382024-08-26T20:34:50.272ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
34392024-08-26T20:34:50.273ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
34402024-08-26T20:34:50.274ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
34412024-08-26T20:34:50.274ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
34422024-08-26T20:34:50.275ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
34432024-08-26T20:34:50.277ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: c9d3ee27-fe3f-422b-93ce-695a8356d9d2, lookup_type: ById(c9d3ee27-fe3f-422b-93ce-695a8356d9d2) } result = Ok(())
34442024-08-26T20:34:50.277ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
34452024-08-26T20:34:50.277ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
34462024-08-26T20:34:50.278ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 29.665898ms iteration = 14 status = 0
34472024-08-26T20:34:50.278ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
34482024-08-26T20:34:50.279ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
34492024-08-26T20:34:50.279ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 30.183602ms iteration = 14 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
34502024-08-26T20:34:50.419ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
34512024-08-26T20:34:50.420ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
34522024-08-26T20:34:50.421ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "766b9166-bbf4-4e43-9bf2-92a8e64368b0", "content-length": "26", "date": "Mon, 26 Aug 2024 20:34:50 GMT"} })
34532024-08-26T20:34:50.421ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
34542024-08-26T20:34:50.423ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
34552024-08-26T20:34:50.438ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
34562024-08-26T20:34:50.438ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "cbcdb6b7-a31e-4ff2-a583-e2732a06e892", "content-length": "585", "date": "Mon, 26 Aug 2024 20:34:50 GMT"} })
34572024-08-26T20:34:50.438ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
34582024-08-26T20:34:50.439ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
34592024-08-26T20:34:50.440ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
34602024-08-26T20:34:50.440ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
34612024-08-26T20:34:50.442ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
34622024-08-26T20:34:50.442ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
34632024-08-26T20:34:50.443ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
34642024-08-26T20:34:50.444ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
34652024-08-26T20:34:50.444ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
34662024-08-26T20:34:50.445ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
34672024-08-26T20:34:50.446ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
34682024-08-26T20:34:50.446ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
34692024-08-26T20:34:50.448ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
34702024-08-26T20:34:50.448ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 204, headers: {"x-request-id": "3db0bb3f-7ac3-48ef-a67a-3f09882365d2", "date": "Mon, 26 Aug 2024 20:34:50 GMT"} })
34712024-08-26T20:34:50.449ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 200.033691ms iteration = 14 status = {}
34722024-08-26T20:34:50.507ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
34732024-08-26T20:34:50.507ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
34742024-08-26T20:34:50.507ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "3039afbf-26ca-4288-940b-75ac56bf0b4f", "content-length": "26", "date": "Mon, 26 Aug 2024 20:34:50 GMT"} })
34752024-08-26T20:34:50.507ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
34762024-08-26T20:34:50.509ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
34772024-08-26T20:34:50.547ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
34782024-08-26T20:34:50.548ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/v2p", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "e91c031f-ed3a-4e6c-af78-e9b3ccf3b5f3", "content-length": "811", "date": "Mon, 26 Aug 2024 20:34:50 GMT"} })
34792024-08-26T20:34:50.548ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {VirtualNetworkInterfaceHost { physical_host_ip: fd00:1122:3344:101::1, virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 246, 67, 165])), vni: Vni(11448629) }} sled = sock
34802024-08-26T20:34:50.548ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = sock
34812024-08-26T20:34:50.548ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = sock
34822024-08-26T20:34:50.549ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 300.998665ms iteration = 14 status = {}
34832024-08-26T20:34:50.550ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1058 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 zone_address = fd00:1122:3344:101::2
34842024-08-26T20:34:50.550ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
34852024-08-26T20:34:50.551ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "918510e8-44fd-4f8c-a66c-348793805c0f", "content-length": "26", "date": "Mon, 26 Aug 2024 20:34:50 GMT"} })
34862024-08-26T20:34:50.551ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1063 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
34872024-08-26T20:34:50.553ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
34882024-08-26T20:34:50.565ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = instance_watcher body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/instances/27ce8152-df37-4e7b-86ff-b2cbe645ab2f/state
34892024-08-26T20:34:50.565ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = instance_watcher result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/instances/27ce8152-df37-4e7b-86ff-b2cbe645ab2f/state", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "ee86822c-c100-476e-98aa-d6ce28c84914", "content-length": "183", "date": "Mon, 26 Aug 2024 20:34:50 GMT"} })
34902024-08-26T20:34:50.566ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
34912024-08-26T20:34:50.567ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "dacc7812-9c9e-4c21-8c74-138b988e99e6", "content-length": "1", "date": "Mon, 26 Aug 2024 20:34:50 GMT"} })
34922024-08-26T20:34:50.567ZDEBGnexus (BackgroundTasks): updating instance state background_task = instance_watcher instance_id = 27ce8152-df37-4e7b-86ff-b2cbe645ab2f state = Running
34932024-08-26T20:34:50.569ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
34942024-08-26T20:34:50.569ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
34952024-08-26T20:34:50.571ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 322.683088ms iteration = 14 status = null
34962024-08-26T20:34:50.572ZINFOnexus (BackgroundTasks): received new VMM runtime state from sled agent background_task = instance_watcher file = nexus/src/app/instance.rs:1846 instance_id = 27ce8152-df37-4e7b-86ff-b2cbe645ab2f migration_state = Migrations { migration_in: None, migration_out: None } propolis_id = 16a33d62-43a6-46fc-b522-532e5def8a45 vmm_state = VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-08-26T20:34:43.655612982Z }
34972024-08-26T20:34:50.572ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher instance_id = 27ce8152-df37-4e7b-86ff-b2cbe645ab2f roles = RoleSet { roles: {} }
34982024-08-26T20:34:50.572ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher instance_id = 27ce8152-df37-4e7b-86ff-b2cbe645ab2f resource = Database result = Ok(())
34992024-08-26T20:34:50.588ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:454 pruned_instances = 0 total_completed = 1 total_failed = 0 total_incomplete = 0 total_instances = 1 update_sagas_queued = 0
35002024-08-26T20:34:50.588ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 338.723967ms iteration = 14 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{"running":1},"pruned_instances":0,"total_instances":1,"update_sagas_queued":0}
35012024-08-26T20:34:50.614ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 365.450887ms iteration = 14 status = {}
35022024-08-26T20:34:50.677ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
35032024-08-26T20:34:50.677ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
35042024-08-26T20:34:50.679ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:147 port = SwitchPort { id: 77948b01-64a9-403e-be7b-4ef5e426dda4, rack_id: 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(735476d6-05ca-4fbd-a182-ab082f700355) } rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
35052024-08-26T20:34:50.679ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
35062024-08-26T20:34:50.679ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
35072024-08-26T20:34:50.687ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
35082024-08-26T20:34:50.687ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/port/qsfp0/settings", query: Some("tag=omicron"), fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "1d8356e2-bd05-446e-82be-1994fd651380", "content-length": "121", "date": "Mon, 26 Aug 2024 20:34:50 GMT"} })
35092024-08-26T20:34:50.687ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1714 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.2.174], params: LinkCreate { autoneg: false, fec: None, kr: false, lane: Some(LinkId(0)), speed: Speed40G } }} } switch_port_id = "qsfp0"
35102024-08-26T20:34:50.687ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1741 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = 735476d6-05ca-4fbd-a182-ab082f700355
35112024-08-26T20:34:50.687ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:379 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
35122024-08-26T20:34:50.687ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
35132024-08-26T20:34:50.688ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:392 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
35142024-08-26T20:34:50.688ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:398 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
35152024-08-26T20:34:50.688ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:418 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
35162024-08-26T20:34:50.688ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
35172024-08-26T20:34:50.688ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
35182024-08-26T20:34:50.689ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
35192024-08-26T20:34:50.689ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv4", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "acc92935-4efb-4d43-ae70-d40aa404cda2", "content-length": "2", "date": "Mon, 26 Aug 2024 20:34:50 GMT"} })
35202024-08-26T20:34:50.689ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
35212024-08-26T20:34:50.689ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv6", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "5da81e1a-92cc-47dd-8ad5-86d5826435bc", "content-length": "52", "date": "Mon, 26 Aug 2024 20:34:50 GMT"} })
35222024-08-26T20:34:50.689ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0" }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:467 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
35232024-08-26T20:34:50.689ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
35242024-08-26T20:34:50.781ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/switch-ports", query: None, fragment: None }, status: 204, headers: {"x-request-id": "3dc651fb-8b1a-4600-9867-3ba42d8d98a6", "date": "Mon, 26 Aug 2024 20:34:50 GMT"} })
35252024-08-26T20:34:50.781ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:770 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch = Switch0
35262024-08-26T20:34:50.781ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
35272024-08-26T20:34:50.781ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/network-bootstore-config", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "a68fcde5-981a-4ecf-bcee-bbe9aeafff70", "content-length": "667", "date": "Mon, 26 Aug 2024 20:34:50 GMT"} })
35282024-08-26T20:34:50.781ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
35292024-08-26T20:34:50.781ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
35302024-08-26T20:34:50.784ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
35312024-08-26T20:34:50.784ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
35322024-08-26T20:34:50.787ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
35332024-08-26T20:34:50.787ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
35342024-08-26T20:34:50.788ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
35352024-08-26T20:34:50.788ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
35362024-08-26T20:34:50.794ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 546.105487ms iteration = 14 status = {}
35372024-08-26T20:34:51.095ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 282 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 14ac83be-ba2a-48de-a061-3ffeed0976e7 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
35382024-08-26T20:35:00.248ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 41 reason = Timeout
35392024-08-26T20:35:00.248ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
35402024-08-26T20:35:00.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
35412024-08-26T20:35:00.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 41 reason = Timeout
35422024-08-26T20:35:00.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
35432024-08-26T20:35:00.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
35442024-08-26T20:35:00.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
35452024-08-26T20:35:00.251ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
35462024-08-26T20:35:00.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
35472024-08-26T20:35:00.252ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 2.525934ms iteration = 41 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
35482024-08-26T20:35:00.253ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
35492024-08-26T20:35:00.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
35502024-08-26T20:35:00.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
35512024-08-26T20:35:00.255ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
35522024-08-26T20:35:00.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
35532024-08-26T20:35:00.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
35542024-08-26T20:35:00.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
35552024-08-26T20:35:00.258ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
35562024-08-26T20:35:00.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
35572024-08-26T20:35:00.264ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
35582024-08-26T20:35:00.264ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 15.96713ms iteration = 41 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
35592024-08-26T20:35:01.095ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 435 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 013d4d58-b117-4e33-9e58-c0e0ef045e6e response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
35602024-08-26T20:35:10.249ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 42 reason = Timeout
35612024-08-26T20:35:10.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
35622024-08-26T20:35:10.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
35632024-08-26T20:35:10.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 42 reason = Timeout
35642024-08-26T20:35:10.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
35652024-08-26T20:35:10.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
35662024-08-26T20:35:10.252ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
35672024-08-26T20:35:10.252ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
35682024-08-26T20:35:10.252ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
35692024-08-26T20:35:10.253ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 3.644824ms iteration = 42 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
35702024-08-26T20:35:10.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
35712024-08-26T20:35:10.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
35722024-08-26T20:35:10.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
35732024-08-26T20:35:10.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
35742024-08-26T20:35:10.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
35752024-08-26T20:35:10.257ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
35762024-08-26T20:35:10.259ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
35772024-08-26T20:35:10.259ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
35782024-08-26T20:35:10.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
35792024-08-26T20:35:10.265ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
35802024-08-26T20:35:10.265ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 16.470334ms iteration = 42 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
35812024-08-26T20:35:11.095ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 473 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = de809c0f-ee64-4791-ad94-367925eb456d response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
35822024-08-26T20:35:20.248ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 10 reason = Timeout
35832024-08-26T20:35:20.248ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 8 reason = Timeout
35842024-08-26T20:35:20.248ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 8 reason = Timeout
35852024-08-26T20:35:20.248ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 15 reason = Timeout
35862024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
35872024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
35882024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 15 reason = Timeout
35892024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 15 reason = Timeout
35902024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 15 reason = Timeout
35912024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
35922024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
35932024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 8 reason = Timeout
35942024-08-26T20:35:20.249ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-08-26 20:25:20.249022347 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
35952024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
35962024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 16 reason = Timeout
35972024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 15 reason = Timeout
35982024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
35992024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
36002024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 43 reason = Timeout
36012024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 8 reason = Timeout
36022024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 10 reason = Timeout
36032024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
36042024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 8 reason = Timeout
36052024-08-26T20:35:20.249ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
36062024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
36072024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 9 reason = Timeout
36082024-08-26T20:35:20.249ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:80 reason = blueprint disabled target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460
36092024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 17.74\u{b5}s iteration = 9 status = {"error":"blueprint disabled","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460"}
36102024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 15 reason = Timeout
36112024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 8 reason = Timeout
36122024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
36132024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
36142024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
36152024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 8 reason = Timeout
36162024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
36172024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
36182024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 15 reason = Timeout
36192024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
36202024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
36212024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
36222024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
36232024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
36242024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
36252024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 15 reason = Timeout
36262024-08-26T20:35:20.249ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task started background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:417
36272024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 43 reason = Timeout
36282024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
36292024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 15 reason = Timeout
36302024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
36312024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
36322024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
36332024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
36342024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 15 reason = Timeout
36352024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
36362024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 15 reason = Timeout
36372024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
36382024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 15 reason = Timeout
36392024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
36402024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
36412024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 8 reason = Timeout
36422024-08-26T20:35:20.249ZINFOnexus (BackgroundTasks): Abandoned VMM reaper running background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:76
36432024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 15 reason = Timeout
36442024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
36452024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
36462024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
36472024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 8 reason = Timeout
36482024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
36492024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
36502024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
36512024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
36522024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
36532024-08-26T20:35:20.249ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("be00167b-db1f-4cc1-86f1-a2af7f1c31f8.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("d2b5c33b-9149-4357-86b1-60e922261ec4.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("d2b5c33b-9149-4357-86b1-60e922261ec4.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 1219, tv_nsec: 423845044 } })
36542024-08-26T20:35:20.249ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("316f64f2-8fa5-43f3-8d5d-92212e3046c4.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("35cb7d67-b9f1-4dcf-bac3-28244fb25d88.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("e37633b8-b58b-40c2-b3c2-6873358565b7.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("e37633b8-b58b-40c2-b3c2-6873358565b7.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:3::1))) }], valid_until: Instant { tv_sec: 1219, tv_nsec: 423850764 } })
36552024-08-26T20:35:20.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
36562024-08-26T20:35:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
36572024-08-26T20:35:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
36582024-08-26T20:35:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
36592024-08-26T20:35:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
36602024-08-26T20:35:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
36612024-08-26T20:35:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
36622024-08-26T20:35:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
36632024-08-26T20:35:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
36642024-08-26T20:35:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
36652024-08-26T20:35:20.251ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
36662024-08-26T20:35:20.251ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
36672024-08-26T20:35:20.251ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 2.612184ms iteration = 8 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
36682024-08-26T20:35:20.251ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } dns_group = internal
36692024-08-26T20:35:20.251ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 2.433652ms iteration = 8 status = {"addresses":["[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353","[fd00:1122:3344:3::1]:5353"]}
36702024-08-26T20:35:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
36712024-08-26T20:35:20.251ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
36722024-08-26T20:35:20.252ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
36732024-08-26T20:35:20.252ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
36742024-08-26T20:35:20.252ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-08-26 20:25:20.249022347 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
36752024-08-26T20:35:20.252ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 3.662544ms iteration = 8 status = {"expiration":"2024-08-26T20:25:20.249022347Z","pruned":[]}
36762024-08-26T20:35:20.252ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
36772024-08-26T20:35:20.252ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.852886ms iteration = 15 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
36782024-08-26T20:35:20.252ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
36792024-08-26T20:35:20.253ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
36802024-08-26T20:35:20.253ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1219, tv_nsec: 427563889 } })
36812024-08-26T20:35:20.254ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
36822024-08-26T20:35:20.254ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
36832024-08-26T20:35:20.254ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
36842024-08-26T20:35:20.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
36852024-08-26T20:35:20.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
36862024-08-26T20:35:20.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
36872024-08-26T20:35:20.254ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
36882024-08-26T20:35:20.254ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1050
36892024-08-26T20:35:20.254ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1219, tv_nsec: 428857821 } })
36902024-08-26T20:35:20.255ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
36912024-08-26T20:35:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
36922024-08-26T20:35:20.255ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
36932024-08-26T20:35:20.255ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
36942024-08-26T20:35:20.255ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
36952024-08-26T20:35:20.255ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
36962024-08-26T20:35:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
36972024-08-26T20:35:20.255ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1050
36982024-08-26T20:35:20.255ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 6.035846ms iteration = 15 status = {"region_replacement_started_err":0,"region_replacement_started_ok":0}
36992024-08-26T20:35:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
37002024-08-26T20:35:20.255ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
37012024-08-26T20:35:20.255ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
37022024-08-26T20:35:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
37032024-08-26T20:35:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
37042024-08-26T20:35:20.255ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 6.129397ms iteration = 15 status = {"errors":[],"garbage_collect_requested":[]}
37052024-08-26T20:35:20.256ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
37062024-08-26T20:35:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
37072024-08-26T20:35:20.256ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0
37082024-08-26T20:35:20.256ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:189 found = 0 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0
37092024-08-26T20:35:20.256ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 6.667992ms iteration = 8 status = {"error":null,"error_count":0,"found":0,"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0}
37102024-08-26T20:35:20.256ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
37112024-08-26T20:35:20.257ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
37122024-08-26T20:35:20.257ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
37132024-08-26T20:35:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
37142024-08-26T20:35:20.257ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
37152024-08-26T20:35:20.257ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
37162024-08-26T20:35:20.257ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
37172024-08-26T20:35:20.257ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
37182024-08-26T20:35:20.257ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:220 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
37192024-08-26T20:35:20.257ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 7.828523ms iteration = 15 status = {"destroyed_active_vmms":0,"error":null,"saga_start_failures":0,"sagas_completed":0,"sagas_failed":0,"sagas_started":0,"terminated_active_migrations":0}
37202024-08-26T20:35:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
37212024-08-26T20:35:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
37222024-08-26T20:35:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
37232024-08-26T20:35:20.258ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
37242024-08-26T20:35:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
37252024-08-26T20:35:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
37262024-08-26T20:35:20.258ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1219, tv_nsec: 432291453 } })
37272024-08-26T20:35:20.258ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1050 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
37282024-08-26T20:35:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
37292024-08-26T20:35:20.258ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
37302024-08-26T20:35:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
37312024-08-26T20:35:20.259ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
37322024-08-26T20:35:20.259ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
37332024-08-26T20:35:20.259ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
37342024-08-26T20:35:20.259ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
37352024-08-26T20:35:20.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
37362024-08-26T20:35:20.259ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
37372024-08-26T20:35:20.259ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
37382024-08-26T20:35:20.259ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 10.147055ms iteration = 8 status = {"errors":[],"found_port_ok":[]}
37392024-08-26T20:35:20.260ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
37402024-08-26T20:35:20.260ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
37412024-08-26T20:35:20.260ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
37422024-08-26T20:35:20.261ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
37432024-08-26T20:35:20.261ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
37442024-08-26T20:35:20.261ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
37452024-08-26T20:35:20.261ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
37462024-08-26T20:35:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
37472024-08-26T20:35:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
37482024-08-26T20:35:20.262ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
37492024-08-26T20:35:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
37502024-08-26T20:35:20.262ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
37512024-08-26T20:35:20.262ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
37522024-08-26T20:35:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
37532024-08-26T20:35:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
37542024-08-26T20:35:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
37552024-08-26T20:35:20.262ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
37562024-08-26T20:35:20.262ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
37572024-08-26T20:35:20.262ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
37582024-08-26T20:35:20.262ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
37592024-08-26T20:35:20.262ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
37602024-08-26T20:35:20.262ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
37612024-08-26T20:35:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
37622024-08-26T20:35:20.262ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 13.173494ms iteration = 15 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
37632024-08-26T20:35:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
37642024-08-26T20:35:20.264ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
37652024-08-26T20:35:20.264ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
37662024-08-26T20:35:20.264ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
37672024-08-26T20:35:20.264ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
37682024-08-26T20:35:20.264ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
37692024-08-26T20:35:20.264ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
37702024-08-26T20:35:20.264ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
37712024-08-26T20:35:20.264ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
37722024-08-26T20:35:20.265ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
37732024-08-26T20:35:20.265ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = b82a4171-23b2-444b-9597-09085a7a2fb8 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
37742024-08-26T20:35:20.265ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
37752024-08-26T20:35:20.265ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
37762024-08-26T20:35:20.265ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
37772024-08-26T20:35:20.265ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 16.593015ms iteration = 8 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
37782024-08-26T20:35:20.265ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
37792024-08-26T20:35:20.265ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
37802024-08-26T20:35:20.266ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
37812024-08-26T20:35:20.266ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
37822024-08-26T20:35:20.266ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 17.485034ms iteration = 16 status = {"physical_disks_added":0}
37832024-08-26T20:35:20.266ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
37842024-08-26T20:35:20.266ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
37852024-08-26T20:35:20.266ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
37862024-08-26T20:35:20.267ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
37872024-08-26T20:35:20.267ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
37882024-08-26T20:35:20.267ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
37892024-08-26T20:35:20.267ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
37902024-08-26T20:35:20.267ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
37912024-08-26T20:35:20.267ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
37922024-08-26T20:35:20.268ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 18.356462ms iteration = 43 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
37932024-08-26T20:35:20.268ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
37942024-08-26T20:35:20.268ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = d840a78c-2f0d-4cdf-9962-31a3dae384a2 dns_zone_name = oxide.test found_so_far = 0 version = 2
37952024-08-26T20:35:20.268ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
37962024-08-26T20:35:20.268ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
37972024-08-26T20:35:20.268ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:454 pruned_instances = 1 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
37982024-08-26T20:35:20.268ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 19.195859ms iteration = 15 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":1,"total_instances":0,"update_sagas_queued":0}
37992024-08-26T20:35:20.268ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
38002024-08-26T20:35:20.269ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
38012024-08-26T20:35:20.269ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
38022024-08-26T20:35:20.269ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
38032024-08-26T20:35:20.269ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
38042024-08-26T20:35:20.269ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
38052024-08-26T20:35:20.269ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
38062024-08-26T20:35:20.269ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
38072024-08-26T20:35:20.270ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
38082024-08-26T20:35:20.270ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
38092024-08-26T20:35:20.271ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = b82a4171-23b2-444b-9597-09085a7a2fb8 dns_zone_name = control-plane.oxide.internal found_so_far = 50 version = 1
38102024-08-26T20:35:20.271ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
38112024-08-26T20:35:20.271ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-08-26 20:29:01.089496 UTC dns_group = internal generation = 1
38122024-08-26T20:35:20.271ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 22.569211ms iteration = 8 status = {"generation":1}
38132024-08-26T20:35:20.272ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
38142024-08-26T20:35:20.272ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
38152024-08-26T20:35:20.272ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
38162024-08-26T20:35:20.272ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
38172024-08-26T20:35:20.273ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
38182024-08-26T20:35:20.273ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
38192024-08-26T20:35:20.273ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
38202024-08-26T20:35:20.273ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
38212024-08-26T20:35:20.273ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
38222024-08-26T20:35:20.273ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
38232024-08-26T20:35:20.274ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
38242024-08-26T20:35:20.274ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
38252024-08-26T20:35:20.274ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
38262024-08-26T20:35:20.274ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = d840a78c-2f0d-4cdf-9962-31a3dae384a2 dns_zone_name = oxide.test found_so_far = 1 version = 2
38272024-08-26T20:35:20.274ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
38282024-08-26T20:35:20.274ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-08-26 20:29:01.487349 UTC dns_group = external generation = 2
38292024-08-26T20:35:20.274ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 25.482559ms iteration = 8 status = {"generation":2}
38302024-08-26T20:35:20.275ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
38312024-08-26T20:35:20.275ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
38322024-08-26T20:35:20.275ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
38332024-08-26T20:35:20.275ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
38342024-08-26T20:35:20.276ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
38352024-08-26T20:35:20.278ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: c9d3ee27-fe3f-422b-93ce-695a8356d9d2, lookup_type: ById(c9d3ee27-fe3f-422b-93ce-695a8356d9d2) } result = Ok(())
38362024-08-26T20:35:20.278ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
38372024-08-26T20:35:20.278ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
38382024-08-26T20:35:20.278ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
38392024-08-26T20:35:20.278ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
38402024-08-26T20:35:20.278ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
38412024-08-26T20:35:20.278ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 29.255054ms iteration = 43 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
38422024-08-26T20:35:20.278ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
38432024-08-26T20:35:20.279ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 30.411205ms iteration = 15 status = 0
38442024-08-26T20:35:20.282ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task done background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:439
38452024-08-26T20:35:20.282ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 33.10456ms iteration = 15 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
38462024-08-26T20:35:20.308ZINFOnexus (BackgroundTasks): found Silo/DNS/TLS config (changed) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-08-26T20:28:18.089116Z, time_modified: 2024-08-26T20:28:18.089116Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 80a72585-f38a-4ccb-b69a-caafab415ca2, db_silo: Silo { identity: SiloIdentity { id: 80a72585-f38a-4ccb-b69a-caafab415ca2, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-08-26T20:29:01.450089Z, time_modified: 2024-08-26T20:29:01.450089Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(4)) }, tls_certs: [TlsCertificate { digest: "81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 80a72585-f38a-4ccb-b69a-caafab415ca2, db_silo: Silo { identity: SiloIdentity { id: 80a72585-f38a-4ccb-b69a-caafab415ca2, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-08-26T20:29:01.450089Z, time_modified: 2024-08-26T20:29:01.450089Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(4)) }, tls_certs: [TlsCertificate { digest: "81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923" }] }) } file = nexus/src/app/background/tasks/external_endpoints.rs:94
38472024-08-26T20:35:20.308ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 59.087633ms iteration = 8 status = {"by_dns_name":{"default-silo.sys.oxide.test":{"silo_id":"001de000-5110-4000-8000-000000000000","tls_certs":[]},"recovery.sys.oxide.test":{"silo_id":"80a72585-f38a-4ccb-b69a-caafab415ca2","tls_certs":["81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923"]}},"default_endpoint":{"silo_id":"80a72585-f38a-4ccb-b69a-caafab415ca2","tls_certs":["81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923"]},"warnings":["silo 001de000-5110-4000-8000-000000000000 with DNS name \\"default-silo.sys.oxide.test\\" has no usable certificates"]}
38482024-08-26T20:35:20.409ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
38492024-08-26T20:35:20.410ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "a1f7829d-4a5b-4c68-875a-30625a9ca1ee", "content-length": "372", "date": "Mon, 26 Aug 2024 20:35:19 GMT"} })
38502024-08-26T20:35:20.410ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
38512024-08-26T20:35:20.410ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
38522024-08-26T20:35:20.412ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
38532024-08-26T20:35:20.412ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
38542024-08-26T20:35:20.414ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
38552024-08-26T20:35:20.414ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
38562024-08-26T20:35:20.416ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
38572024-08-26T20:35:20.416ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 204, headers: {"x-request-id": "000a962c-b8b7-445e-8be6-b0c3f0f3cf01", "date": "Mon, 26 Aug 2024 20:35:19 GMT"} })
38582024-08-26T20:35:20.419ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 169.548135ms iteration = 15 status = {}
38592024-08-26T20:35:20.460ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1058 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 zone_address = fd00:1122:3344:101::2
38602024-08-26T20:35:20.460ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
38612024-08-26T20:35:20.460ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "b0577171-b2cc-4d47-a748-ba42ac0caa3d", "content-length": "26", "date": "Mon, 26 Aug 2024 20:35:20 GMT"} })
38622024-08-26T20:35:20.460ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1063 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
38632024-08-26T20:35:20.462ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
38642024-08-26T20:35:20.473ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
38652024-08-26T20:35:20.473ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
38662024-08-26T20:35:20.473ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "c5eabb7f-4f7f-41f3-a61d-25a9f42b3e13", "content-length": "26", "date": "Mon, 26 Aug 2024 20:35:20 GMT"} })
38672024-08-26T20:35:20.473ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
38682024-08-26T20:35:20.474ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
38692024-08-26T20:35:20.475ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
38702024-08-26T20:35:20.475ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
38712024-08-26T20:35:20.476ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "fb461ce1-00e9-41fe-929e-32de538fb3e6", "content-length": "26", "date": "Mon, 26 Aug 2024 20:35:20 GMT"} })
38722024-08-26T20:35:20.476ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
38732024-08-26T20:35:20.477ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
38742024-08-26T20:35:20.500ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
38752024-08-26T20:35:20.500ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/v2p", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "3fcc4adc-c389-4499-bae5-12bc917c28b7", "content-length": "691", "date": "Mon, 26 Aug 2024 20:35:19 GMT"} })
38762024-08-26T20:35:20.500ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = sock
38772024-08-26T20:35:20.500ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = sock
38782024-08-26T20:35:20.500ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = sock
38792024-08-26T20:35:20.501ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 252.510752ms iteration = 15 status = {}
38802024-08-26T20:35:20.538ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
38812024-08-26T20:35:20.549ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
38822024-08-26T20:35:20.599ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
38832024-08-26T20:35:20.599ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "c26d5b9d-5b89-46fc-aa24-c6955bfcce7f", "content-length": "2", "date": "Mon, 26 Aug 2024 20:35:20 GMT"} })
38842024-08-26T20:35:20.601ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
38852024-08-26T20:35:20.601ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
38862024-08-26T20:35:20.602ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 353.769818ms iteration = 15 status = null
38872024-08-26T20:35:20.607ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 358.722764ms iteration = 15 status = {}
38882024-08-26T20:35:20.731ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
38892024-08-26T20:35:20.731ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::9)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "f1aa1d36-4d73-4d17-b0fe-17ed4df4623a", "date": "Mon, 26 Aug 2024 20:35:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
38902024-08-26T20:35:20.731ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
38912024-08-26T20:35:20.733ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::8)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "f3f15bad-e94e-4d28-b594-9816cddb1e0c", "date": "Mon, 26 Aug 2024 20:35:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
38922024-08-26T20:35:20.733ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
38932024-08-26T20:35:20.735ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
38942024-08-26T20:35:20.735ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 486.268388ms iteration = 10 status = {"generation":2,"server_results":{"[fd00:1122:3344:101::8]:5353":{"Ok":null},"[fd00:1122:3344:101::9]:5353":{"Ok":null}}}
38952024-08-26T20:35:20.744ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
38962024-08-26T20:35:20.769ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
38972024-08-26T20:35:20.769ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
38982024-08-26T20:35:20.771ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:147 port = SwitchPort { id: 77948b01-64a9-403e-be7b-4ef5e426dda4, rack_id: 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(735476d6-05ca-4fbd-a182-ab082f700355) } rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
38992024-08-26T20:35:20.771ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
39002024-08-26T20:35:20.771ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
39012024-08-26T20:35:20.778ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
39022024-08-26T20:35:20.778ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/port/qsfp0/settings", query: Some("tag=omicron"), fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "ccfb7b56-4b76-4c71-98a5-86309789db61", "content-length": "121", "date": "Mon, 26 Aug 2024 20:35:20 GMT"} })
39032024-08-26T20:35:20.778ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1714 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.2.174], params: LinkCreate { autoneg: false, fec: None, kr: false, lane: Some(LinkId(0)), speed: Speed40G } }} } switch_port_id = "qsfp0"
39042024-08-26T20:35:20.778ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1741 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = 735476d6-05ca-4fbd-a182-ab082f700355
39052024-08-26T20:35:20.779ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:379 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
39062024-08-26T20:35:20.779ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
39072024-08-26T20:35:20.779ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:392 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
39082024-08-26T20:35:20.779ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:398 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
39092024-08-26T20:35:20.779ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:418 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
39102024-08-26T20:35:20.779ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
39112024-08-26T20:35:20.779ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
39122024-08-26T20:35:20.780ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
39132024-08-26T20:35:20.781ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv4", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "31fc12eb-3acc-48f5-adc8-6e6a3b6821fc", "content-length": "2", "date": "Mon, 26 Aug 2024 20:35:20 GMT"} })
39142024-08-26T20:35:20.781ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
39152024-08-26T20:35:20.781ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv6", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "e4095dab-1ad1-4f7b-9e41-4aba8b27e79b", "content-length": "52", "date": "Mon, 26 Aug 2024 20:35:20 GMT"} })
39162024-08-26T20:35:20.781ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0" }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:467 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
39172024-08-26T20:35:20.781ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
39182024-08-26T20:35:20.788ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
39192024-08-26T20:35:20.788ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:1::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "2e3edc90-605b-4579-990b-3f718021789d", "date": "Mon, 26 Aug 2024 20:35:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
39202024-08-26T20:35:20.788ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
39212024-08-26T20:35:20.790ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:2::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "750ab29b-02c3-4b3a-9478-aa1ed21966b9", "date": "Mon, 26 Aug 2024 20:35:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
39222024-08-26T20:35:20.790ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
39232024-08-26T20:35:20.791ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:3::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "73bf67f3-3288-4856-a7e7-b15fd7955fb4", "date": "Mon, 26 Aug 2024 20:35:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
39242024-08-26T20:35:20.791ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
39252024-08-26T20:35:20.793ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
39262024-08-26T20:35:20.793ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 544.167919ms iteration = 10 status = {"generation":1,"server_results":{"[fd00:1122:3344:1::1]:5353":{"Ok":null},"[fd00:1122:3344:2::1]:5353":{"Ok":null},"[fd00:1122:3344:3::1]:5353":{"Ok":null}}}
39272024-08-26T20:35:20.869ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/switch-ports", query: None, fragment: None }, status: 204, headers: {"x-request-id": "6dc5ef79-8ed1-40ad-87bd-743f11bb1e31", "date": "Mon, 26 Aug 2024 20:35:20 GMT"} })
39282024-08-26T20:35:20.869ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:770 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch = Switch0
39292024-08-26T20:35:20.869ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
39302024-08-26T20:35:20.870ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/network-bootstore-config", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "71b75130-87f6-436d-8a0d-b191d69373a6", "content-length": "667", "date": "Mon, 26 Aug 2024 20:35:20 GMT"} })
39312024-08-26T20:35:20.870ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
39322024-08-26T20:35:20.870ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
39332024-08-26T20:35:20.873ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
39342024-08-26T20:35:20.873ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
39352024-08-26T20:35:20.875ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
39362024-08-26T20:35:20.876ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
39372024-08-26T20:35:20.877ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
39382024-08-26T20:35:20.877ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
39392024-08-26T20:35:20.884ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 635.365102ms iteration = 15 status = {}
39402024-08-26T20:35:21.095ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 320 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 58f7a5d7-d741-4d9a-ba86-5e91efc91ff0 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
39412024-08-26T20:35:30.248ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 44 reason = Timeout
39422024-08-26T20:35:30.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
39432024-08-26T20:35:30.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
39442024-08-26T20:35:30.250ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 44 reason = Timeout
39452024-08-26T20:35:30.250ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
39462024-08-26T20:35:30.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
39472024-08-26T20:35:30.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
39482024-08-26T20:35:30.251ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
39492024-08-26T20:35:30.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
39502024-08-26T20:35:30.252ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 2.523574ms iteration = 44 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
39512024-08-26T20:35:30.253ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
39522024-08-26T20:35:30.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
39532024-08-26T20:35:30.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
39542024-08-26T20:35:30.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
39552024-08-26T20:35:30.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
39562024-08-26T20:35:30.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
39572024-08-26T20:35:30.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
39582024-08-26T20:35:30.258ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
39592024-08-26T20:35:30.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
39602024-08-26T20:35:30.264ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
39612024-08-26T20:35:30.264ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 16.02349ms iteration = 44 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
39622024-08-26T20:35:31.096ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 242 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = d795b86d-13cd-4a00-99d1-051dd47c0187 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
39632024-08-26T20:35:40.249ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 45 reason = Timeout
39642024-08-26T20:35:40.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
39652024-08-26T20:35:40.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
39662024-08-26T20:35:40.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 45 reason = Timeout
39672024-08-26T20:35:40.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
39682024-08-26T20:35:40.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
39692024-08-26T20:35:40.251ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
39702024-08-26T20:35:40.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
39712024-08-26T20:35:40.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
39722024-08-26T20:35:40.253ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 3.088869ms iteration = 45 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
39732024-08-26T20:35:40.253ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
39742024-08-26T20:35:40.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
39752024-08-26T20:35:40.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
39762024-08-26T20:35:40.255ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
39772024-08-26T20:35:40.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
39782024-08-26T20:35:40.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
39792024-08-26T20:35:40.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
39802024-08-26T20:35:40.258ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
39812024-08-26T20:35:40.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
39822024-08-26T20:35:40.264ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
39832024-08-26T20:35:40.264ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 15.623746ms iteration = 45 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
39842024-08-26T20:35:41.095ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 338 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 6db137da-e06a-43a7-bba4-f493aec2537a response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
39852024-08-26T20:35:50.249ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 46 reason = Timeout
39862024-08-26T20:35:50.249ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 16 reason = Timeout
39872024-08-26T20:35:50.249ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
39882024-08-26T20:35:50.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
39892024-08-26T20:35:50.249ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 16 reason = Timeout
39902024-08-26T20:35:50.249ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
39912024-08-26T20:35:50.249ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 16 reason = Timeout
39922024-08-26T20:35:50.249ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 17 reason = Timeout
39932024-08-26T20:35:50.249ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
39942024-08-26T20:35:50.249ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 16 reason = Timeout
39952024-08-26T20:35:50.249ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
39962024-08-26T20:35:50.249ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
39972024-08-26T20:35:50.249ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 16 reason = Timeout
39982024-08-26T20:35:50.249ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 16 reason = Timeout
39992024-08-26T20:35:50.249ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 16 reason = Timeout
40002024-08-26T20:35:50.249ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
40012024-08-26T20:35:50.249ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
40022024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
40032024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
40042024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
40052024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
40062024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
40072024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
40082024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
40092024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
40102024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 16 reason = Timeout
40112024-08-26T20:35:50.250ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task started background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:417
40122024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 16 reason = Timeout
40132024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
40142024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
40152024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 16 reason = Timeout
40162024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
40172024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
40182024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 16 reason = Timeout
40192024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 16 reason = Timeout
40202024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
40212024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
40222024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 46 reason = Timeout
40232024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
40242024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 16 reason = Timeout
40252024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
40262024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
40272024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
40282024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
40292024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
40302024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
40312024-08-26T20:35:50.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
40322024-08-26T20:35:50.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
40332024-08-26T20:35:50.252ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
40342024-08-26T20:35:50.252ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
40352024-08-26T20:35:50.253ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.479672ms iteration = 16 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
40362024-08-26T20:35:50.253ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1249, tv_nsec: 426991413 } })
40372024-08-26T20:35:50.253ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
40382024-08-26T20:35:50.253ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
40392024-08-26T20:35:50.253ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1050
40402024-08-26T20:35:50.253ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
40412024-08-26T20:35:50.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
40422024-08-26T20:35:50.254ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
40432024-08-26T20:35:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
40442024-08-26T20:35:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
40452024-08-26T20:35:50.254ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1249, tv_nsec: 428271145 } })
40462024-08-26T20:35:50.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
40472024-08-26T20:35:50.254ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
40482024-08-26T20:35:50.254ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 4.22812ms iteration = 46 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
40492024-08-26T20:35:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
40502024-08-26T20:35:50.255ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
40512024-08-26T20:35:50.255ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
40522024-08-26T20:35:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
40532024-08-26T20:35:50.255ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 6.017766ms iteration = 16 status = {"region_replacement_started_err":0,"region_replacement_started_ok":0}
40542024-08-26T20:35:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
40552024-08-26T20:35:50.255ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1050
40562024-08-26T20:35:50.255ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
40572024-08-26T20:35:50.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
40582024-08-26T20:35:50.256ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 5.723904ms iteration = 16 status = {"errors":[],"garbage_collect_requested":[]}
40592024-08-26T20:35:50.256ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
40602024-08-26T20:35:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
40612024-08-26T20:35:50.256ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
40622024-08-26T20:35:50.256ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
40632024-08-26T20:35:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
40642024-08-26T20:35:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
40652024-08-26T20:35:50.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
40662024-08-26T20:35:50.256ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
40672024-08-26T20:35:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
40682024-08-26T20:35:50.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
40692024-08-26T20:35:50.257ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
40702024-08-26T20:35:50.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
40712024-08-26T20:35:50.257ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1249, tv_nsec: 431254733 } })
40722024-08-26T20:35:50.258ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
40732024-08-26T20:35:50.258ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
40742024-08-26T20:35:50.258ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
40752024-08-26T20:35:50.258ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
40762024-08-26T20:35:50.258ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
40772024-08-26T20:35:50.258ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
40782024-08-26T20:35:50.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
40792024-08-26T20:35:50.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
40802024-08-26T20:35:50.258ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
40812024-08-26T20:35:50.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
40822024-08-26T20:35:50.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
40832024-08-26T20:35:50.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
40842024-08-26T20:35:50.258ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
40852024-08-26T20:35:50.258ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
40862024-08-26T20:35:50.258ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
40872024-08-26T20:35:50.258ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1050 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
40882024-08-26T20:35:50.258ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 8.098876ms iteration = 16 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
40892024-08-26T20:35:50.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
40902024-08-26T20:35:50.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
40912024-08-26T20:35:50.259ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
40922024-08-26T20:35:50.259ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:220 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
40932024-08-26T20:35:50.259ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 9.173996ms iteration = 16 status = {"destroyed_active_vmms":0,"error":null,"saga_start_failures":0,"sagas_completed":0,"sagas_failed":0,"sagas_started":0,"terminated_active_migrations":0}
40942024-08-26T20:35:50.260ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
40952024-08-26T20:35:50.260ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
40962024-08-26T20:35:50.260ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
40972024-08-26T20:35:50.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
40982024-08-26T20:35:50.260ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
40992024-08-26T20:35:50.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
41002024-08-26T20:35:50.261ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
41012024-08-26T20:35:50.261ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
41022024-08-26T20:35:50.262ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
41032024-08-26T20:35:50.262ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
41042024-08-26T20:35:50.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
41052024-08-26T20:35:50.263ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:454 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
41062024-08-26T20:35:50.264ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 13.290674ms iteration = 16 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
41072024-08-26T20:35:50.267ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
41082024-08-26T20:35:50.267ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 18.461213ms iteration = 46 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
41092024-08-26T20:35:50.270ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
41102024-08-26T20:35:50.270ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
41112024-08-26T20:35:50.271ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
41122024-08-26T20:35:50.271ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
41132024-08-26T20:35:50.272ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
41142024-08-26T20:35:50.274ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: c9d3ee27-fe3f-422b-93ce-695a8356d9d2, lookup_type: ById(c9d3ee27-fe3f-422b-93ce-695a8356d9d2) } result = Ok(())
41152024-08-26T20:35:50.274ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
41162024-08-26T20:35:50.274ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
41172024-08-26T20:35:50.275ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 25.787601ms iteration = 16 status = 0
41182024-08-26T20:35:50.287ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task done background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:439
41192024-08-26T20:35:50.287ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 36.557052ms iteration = 16 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
41202024-08-26T20:35:50.288ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 38.763542ms iteration = 17 status = {"physical_disks_added":0}
41212024-08-26T20:35:50.292ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
41222024-08-26T20:35:50.293ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
41232024-08-26T20:35:50.294ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
41242024-08-26T20:35:50.295ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
41252024-08-26T20:35:50.295ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
41262024-08-26T20:35:50.295ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
41272024-08-26T20:35:50.296ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
41282024-08-26T20:35:50.297ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
41292024-08-26T20:35:50.297ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
41302024-08-26T20:35:50.297ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
41312024-08-26T20:35:50.377ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
41322024-08-26T20:35:50.377ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
41332024-08-26T20:35:50.377ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "59c31640-12c4-45d8-8d09-f5512414b6c5", "content-length": "26", "date": "Mon, 26 Aug 2024 20:35:50 GMT"} })
41342024-08-26T20:35:50.377ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
41352024-08-26T20:35:50.379ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
41362024-08-26T20:35:50.386ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
41372024-08-26T20:35:50.386ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "9fdabc90-5dd3-4536-986d-54020a24a55c", "content-length": "372", "date": "Mon, 26 Aug 2024 20:35:50 GMT"} })
41382024-08-26T20:35:50.386ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
41392024-08-26T20:35:50.386ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
41402024-08-26T20:35:50.387ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
41412024-08-26T20:35:50.388ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
41422024-08-26T20:35:50.389ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
41432024-08-26T20:35:50.389ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
41442024-08-26T20:35:50.391ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
41452024-08-26T20:35:50.391ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 204, headers: {"x-request-id": "4be53cb8-b4b0-4410-9924-94c696a0c220", "date": "Mon, 26 Aug 2024 20:35:50 GMT"} })
41462024-08-26T20:35:50.392ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 141.820887ms iteration = 16 status = {}
41472024-08-26T20:35:50.467ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
41482024-08-26T20:35:50.467ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
41492024-08-26T20:35:50.467ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "afa93347-90f0-4811-a650-b6bae30e4c77", "content-length": "26", "date": "Mon, 26 Aug 2024 20:35:50 GMT"} })
41502024-08-26T20:35:50.467ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
41512024-08-26T20:35:50.469ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
41522024-08-26T20:35:50.479ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
41532024-08-26T20:35:50.480ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/v2p", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "52a32585-2046-47bf-bf11-a0c7abad12a1", "content-length": "691", "date": "Mon, 26 Aug 2024 20:35:50 GMT"} })
41542024-08-26T20:35:50.480ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = sock
41552024-08-26T20:35:50.480ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = sock
41562024-08-26T20:35:50.480ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = sock
41572024-08-26T20:35:50.482ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 232.362683ms iteration = 16 status = {}
41582024-08-26T20:35:50.482ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1058 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 zone_address = fd00:1122:3344:101::2
41592024-08-26T20:35:50.482ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
41602024-08-26T20:35:50.483ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "1f11c7f9-c79f-42a4-a211-08df03f2dc3b", "content-length": "26", "date": "Mon, 26 Aug 2024 20:35:50 GMT"} })
41612024-08-26T20:35:50.483ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1063 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
41622024-08-26T20:35:50.485ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
41632024-08-26T20:35:50.515ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
41642024-08-26T20:35:50.515ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "a058f3af-0b55-4891-aea0-3540ffec2515", "content-length": "2", "date": "Mon, 26 Aug 2024 20:35:50 GMT"} })
41652024-08-26T20:35:50.516ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
41662024-08-26T20:35:50.517ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
41672024-08-26T20:35:50.518ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 268.674223ms iteration = 16 status = null
41682024-08-26T20:35:50.537ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 287.772002ms iteration = 16 status = {}
41692024-08-26T20:35:50.605ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
41702024-08-26T20:35:50.606ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
41712024-08-26T20:35:50.607ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:147 port = SwitchPort { id: 77948b01-64a9-403e-be7b-4ef5e426dda4, rack_id: 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(735476d6-05ca-4fbd-a182-ab082f700355) } rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
41722024-08-26T20:35:50.607ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
41732024-08-26T20:35:50.607ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
41742024-08-26T20:35:50.614ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
41752024-08-26T20:35:50.614ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/port/qsfp0/settings", query: Some("tag=omicron"), fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "7ff771aa-d3c1-4303-ab04-b74d6007b7fb", "content-length": "121", "date": "Mon, 26 Aug 2024 20:35:50 GMT"} })
41762024-08-26T20:35:50.614ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1714 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.2.174], params: LinkCreate { autoneg: false, fec: None, kr: false, lane: Some(LinkId(0)), speed: Speed40G } }} } switch_port_id = "qsfp0"
41772024-08-26T20:35:50.614ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1741 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = 735476d6-05ca-4fbd-a182-ab082f700355
41782024-08-26T20:35:50.614ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:379 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
41792024-08-26T20:35:50.614ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
41802024-08-26T20:35:50.614ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:392 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
41812024-08-26T20:35:50.615ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:398 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
41822024-08-26T20:35:50.615ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:418 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
41832024-08-26T20:35:50.615ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
41842024-08-26T20:35:50.615ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
41852024-08-26T20:35:50.616ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
41862024-08-26T20:35:50.617ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv4", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "e38cfa01-afc6-48e5-abc8-060e1f6900a4", "content-length": "2", "date": "Mon, 26 Aug 2024 20:35:50 GMT"} })
41872024-08-26T20:35:50.617ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
41882024-08-26T20:35:50.617ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv6", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "42aa4733-803d-4403-9b3b-d2d3fe550ac4", "content-length": "52", "date": "Mon, 26 Aug 2024 20:35:50 GMT"} })
41892024-08-26T20:35:50.617ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0" }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:467 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
41902024-08-26T20:35:50.617ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
41912024-08-26T20:35:50.706ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/switch-ports", query: None, fragment: None }, status: 204, headers: {"x-request-id": "5e39af1e-b086-4d12-9b2d-77f020e8df6a", "date": "Mon, 26 Aug 2024 20:35:50 GMT"} })
41922024-08-26T20:35:50.706ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:770 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch = Switch0
41932024-08-26T20:35:50.706ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
41942024-08-26T20:35:50.706ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/network-bootstore-config", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "8743eafc-c3cd-4633-9007-fb8d3ffc1c1f", "content-length": "667", "date": "Mon, 26 Aug 2024 20:35:50 GMT"} })
41952024-08-26T20:35:50.706ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
41962024-08-26T20:35:50.707ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
41972024-08-26T20:35:50.709ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
41982024-08-26T20:35:50.709ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
41992024-08-26T20:35:50.712ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
42002024-08-26T20:35:50.713ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
42012024-08-26T20:35:50.714ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
42022024-08-26T20:35:50.714ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
42032024-08-26T20:35:50.721ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 471.451689ms iteration = 16 status = {}
42042024-08-26T20:35:51.096ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 340 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 659bf523-cc4e-494c-8c6f-ae18c297d311 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
42052024-08-26T20:36:00.249ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 47 reason = Timeout
42062024-08-26T20:36:00.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
42072024-08-26T20:36:00.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
42082024-08-26T20:36:00.250ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 47 reason = Timeout
42092024-08-26T20:36:00.250ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
42102024-08-26T20:36:00.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
42112024-08-26T20:36:00.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
42122024-08-26T20:36:00.252ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
42132024-08-26T20:36:00.252ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
42142024-08-26T20:36:00.253ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 2.599665ms iteration = 47 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
42152024-08-26T20:36:00.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
42162024-08-26T20:36:00.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
42172024-08-26T20:36:00.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
42182024-08-26T20:36:00.255ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
42192024-08-26T20:36:00.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
42202024-08-26T20:36:00.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
42212024-08-26T20:36:00.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
42222024-08-26T20:36:00.258ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
42232024-08-26T20:36:00.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
42242024-08-26T20:36:00.265ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
42252024-08-26T20:36:00.265ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 15.824528ms iteration = 47 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
42262024-08-26T20:36:01.096ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 330 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = ed2ba1c2-80c1-4ae4-90af-ec159ed32379 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
42272024-08-26T20:36:10.249ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 48 reason = Timeout
42282024-08-26T20:36:10.249ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
42292024-08-26T20:36:10.249ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
42302024-08-26T20:36:10.250ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 48 reason = Timeout
42312024-08-26T20:36:10.250ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
42322024-08-26T20:36:10.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
42332024-08-26T20:36:10.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
42342024-08-26T20:36:10.251ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
42352024-08-26T20:36:10.252ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
42362024-08-26T20:36:10.253ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 3.120129ms iteration = 48 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
42372024-08-26T20:36:10.253ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
42382024-08-26T20:36:10.253ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
42392024-08-26T20:36:10.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
42402024-08-26T20:36:10.255ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
42412024-08-26T20:36:10.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
42422024-08-26T20:36:10.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
42432024-08-26T20:36:10.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
42442024-08-26T20:36:10.258ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
42452024-08-26T20:36:10.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
42462024-08-26T20:36:10.265ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
42472024-08-26T20:36:10.265ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 15.877189ms iteration = 48 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
42482024-08-26T20:36:11.096ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 333 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 4c758f74-300c-44b7-a0e2-7aa05ef5d0da response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
42492024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 11 reason = Timeout
42502024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 9 reason = Timeout
42512024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 18 reason = Timeout
42522024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
42532024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 10 reason = Timeout
42542024-08-26T20:36:20.250ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:80 reason = blueprint disabled target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460
42552024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 20.011\u{b5}s iteration = 10 status = {"error":"blueprint disabled","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460"}
42562024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 49 reason = Timeout
42572024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 9 reason = Timeout
42582024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
42592024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 9 reason = Timeout
42602024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
42612024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
42622024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 11 reason = Timeout
42632024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 17 reason = Timeout
42642024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 9 reason = Timeout
42652024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
42662024-08-26T20:36:20.250ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
42672024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 9 reason = Timeout
42682024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
42692024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
42702024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
42712024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 17 reason = Timeout
42722024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
42732024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 9 reason = Timeout
42742024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 9 reason = Timeout
42752024-08-26T20:36:20.250ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-08-26 20:26:20.250442994 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
42762024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
42772024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 17 reason = Timeout
42782024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
42792024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 17 reason = Timeout
42802024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
42812024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 17 reason = Timeout
42822024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
42832024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 17 reason = Timeout
42842024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
42852024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 17 reason = Timeout
42862024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
42872024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 17 reason = Timeout
42882024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
42892024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 17 reason = Timeout
42902024-08-26T20:36:20.250ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task started background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:417
42912024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
42922024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 9 reason = Timeout
42932024-08-26T20:36:20.250ZINFOnexus (BackgroundTasks): Abandoned VMM reaper running background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:76
42942024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
42952024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 17 reason = Timeout
42962024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
42972024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
42982024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 49 reason = Timeout
42992024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 17 reason = Timeout
43002024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
43012024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
43022024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 17 reason = Timeout
43032024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 17 reason = Timeout
43042024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 9 reason = Timeout
43052024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
43062024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
43072024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
43082024-08-26T20:36:20.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
43092024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
43102024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
43112024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
43122024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
43132024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
43142024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
43152024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
43162024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
43172024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
43182024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
43192024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
43202024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
43212024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
43222024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
43232024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
43242024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
43252024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
43262024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
43272024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
43282024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
43292024-08-26T20:36:20.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
43302024-08-26T20:36:20.252ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("be00167b-db1f-4cc1-86f1-a2af7f1c31f8.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("d2b5c33b-9149-4357-86b1-60e922261ec4.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("d2b5c33b-9149-4357-86b1-60e922261ec4.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 1279, tv_nsec: 425367198 } })
43312024-08-26T20:36:20.252ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("316f64f2-8fa5-43f3-8d5d-92212e3046c4.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("35cb7d67-b9f1-4dcf-bac3-28244fb25d88.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("e37633b8-b58b-40c2-b3c2-6873358565b7.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("e37633b8-b58b-40c2-b3c2-6873358565b7.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:3::1))) }], valid_until: Instant { tv_sec: 1279, tv_nsec: 425546970 } })
43322024-08-26T20:36:20.252ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
43332024-08-26T20:36:20.253ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
43342024-08-26T20:36:20.253ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
43352024-08-26T20:36:20.253ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
43362024-08-26T20:36:20.253ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
43372024-08-26T20:36:20.253ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.394862ms iteration = 17 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
43382024-08-26T20:36:20.253ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
43392024-08-26T20:36:20.253ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 3.689195ms iteration = 9 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
43402024-08-26T20:36:20.254ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } dns_group = internal
43412024-08-26T20:36:20.254ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 3.792055ms iteration = 9 status = {"addresses":["[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353","[fd00:1122:3344:3::1]:5353"]}
43422024-08-26T20:36:20.254ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
43432024-08-26T20:36:20.254ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
43442024-08-26T20:36:20.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
43452024-08-26T20:36:20.255ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1279, tv_nsec: 428337336 } })
43462024-08-26T20:36:20.255ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
43472024-08-26T20:36:20.255ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1279, tv_nsec: 428419266 } })
43482024-08-26T20:36:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
43492024-08-26T20:36:20.255ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
43502024-08-26T20:36:20.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
43512024-08-26T20:36:20.255ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
43522024-08-26T20:36:20.256ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
43532024-08-26T20:36:20.256ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
43542024-08-26T20:36:20.256ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
43552024-08-26T20:36:20.256ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1050
43562024-08-26T20:36:20.256ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1050
43572024-08-26T20:36:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
43582024-08-26T20:36:20.256ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
43592024-08-26T20:36:20.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
43602024-08-26T20:36:20.256ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
43612024-08-26T20:36:20.256ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
43622024-08-26T20:36:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
43632024-08-26T20:36:20.257ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
43642024-08-26T20:36:20.257ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
43652024-08-26T20:36:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
43662024-08-26T20:36:20.257ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
43672024-08-26T20:36:20.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
43682024-08-26T20:36:20.257ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
43692024-08-26T20:36:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
43702024-08-26T20:36:20.258ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
43712024-08-26T20:36:20.258ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
43722024-08-26T20:36:20.258ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
43732024-08-26T20:36:20.258ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
43742024-08-26T20:36:20.258ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
43752024-08-26T20:36:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
43762024-08-26T20:36:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
43772024-08-26T20:36:20.258ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 7.797653ms iteration = 49 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
43782024-08-26T20:36:20.258ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
43792024-08-26T20:36:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
43802024-08-26T20:36:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
43812024-08-26T20:36:20.258ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
43822024-08-26T20:36:20.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
43832024-08-26T20:36:20.259ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0
43842024-08-26T20:36:20.259ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:189 found = 0 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0
43852024-08-26T20:36:20.259ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 8.624711ms iteration = 9 status = {"error":null,"error_count":0,"found":0,"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0}
43862024-08-26T20:36:20.259ZINFOnexus (BackgroundTasks): region snapshot replacement find affected volumes task done background_task = region_snapshot_replacement_step file = nexus/src/app/background/tasks/region_snapshot_replacement_step.rs:439
43872024-08-26T20:36:20.259ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 8.988884ms iteration = 17 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
43882024-08-26T20:36:20.259ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-08-26 20:26:20.250442994 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
43892024-08-26T20:36:20.259ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 9.461718ms iteration = 9 status = {"expiration":"2024-08-26T20:26:20.250442994Z","pruned":[]}
43902024-08-26T20:36:20.259ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
43912024-08-26T20:36:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
43922024-08-26T20:36:20.260ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
43932024-08-26T20:36:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
43942024-08-26T20:36:20.260ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
43952024-08-26T20:36:20.260ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
43962024-08-26T20:36:20.260ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
43972024-08-26T20:36:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
43982024-08-26T20:36:20.260ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:55
43992024-08-26T20:36:20.260ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:220 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
44002024-08-26T20:36:20.260ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 9.879643ms iteration = 17 status = {"destroyed_active_vmms":0,"error":null,"saga_start_failures":0,"sagas_completed":0,"sagas_failed":0,"sagas_started":0,"terminated_active_migrations":0}
44012024-08-26T20:36:20.260ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
44022024-08-26T20:36:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
44032024-08-26T20:36:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
44042024-08-26T20:36:20.260ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 10.162585ms iteration = 17 status = {"region_replacement_started_err":0,"region_replacement_started_ok":0}
44052024-08-26T20:36:20.260ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
44062024-08-26T20:36:20.260ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
44072024-08-26T20:36:20.260ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
44082024-08-26T20:36:20.260ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
44092024-08-26T20:36:20.260ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
44102024-08-26T20:36:20.261ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
44112024-08-26T20:36:20.261ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 10.827001ms iteration = 17 status = {"errors":[],"garbage_collect_requested":[]}
44122024-08-26T20:36:20.261ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
44132024-08-26T20:36:20.261ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
44142024-08-26T20:36:20.261ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
44152024-08-26T20:36:20.261ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
44162024-08-26T20:36:20.261ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
44172024-08-26T20:36:20.261ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
44182024-08-26T20:36:20.261ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
44192024-08-26T20:36:20.262ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
44202024-08-26T20:36:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
44212024-08-26T20:36:20.262ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
44222024-08-26T20:36:20.262ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 11.874351ms iteration = 9 status = {"errors":[],"found_port_ok":[]}
44232024-08-26T20:36:20.262ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
44242024-08-26T20:36:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
44252024-08-26T20:36:20.262ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
44262024-08-26T20:36:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
44272024-08-26T20:36:20.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
44282024-08-26T20:36:20.262ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
44292024-08-26T20:36:20.262ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
44302024-08-26T20:36:20.262ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
44312024-08-26T20:36:20.263ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
44322024-08-26T20:36:20.263ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
44332024-08-26T20:36:20.263ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
44342024-08-26T20:36:20.263ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 13.035782ms iteration = 9 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
44352024-08-26T20:36:20.263ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
44362024-08-26T20:36:20.264ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
44372024-08-26T20:36:20.264ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
44382024-08-26T20:36:20.264ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
44392024-08-26T20:36:20.264ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
44402024-08-26T20:36:20.265ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
44412024-08-26T20:36:20.265ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
44422024-08-26T20:36:20.265ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 14.470515ms iteration = 17 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
44432024-08-26T20:36:20.265ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
44442024-08-26T20:36:20.266ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
44452024-08-26T20:36:20.266ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
44462024-08-26T20:36:20.266ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
44472024-08-26T20:36:20.266ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
44482024-08-26T20:36:20.266ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
44492024-08-26T20:36:20.266ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
44502024-08-26T20:36:20.267ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-c9d3ee27-fe3f-422b-93ce-695a8356d9d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1279, tv_nsec: 440433859 } })
44512024-08-26T20:36:20.267ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
44522024-08-26T20:36:20.267ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
44532024-08-26T20:36:20.267ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
44542024-08-26T20:36:20.267ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
44552024-08-26T20:36:20.267ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = d840a78c-2f0d-4cdf-9962-31a3dae384a2 dns_zone_name = oxide.test found_so_far = 0 version = 2
44562024-08-26T20:36:20.267ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
44572024-08-26T20:36:20.267ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
44582024-08-26T20:36:20.268ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1050 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
44592024-08-26T20:36:20.268ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
44602024-08-26T20:36:20.268ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
44612024-08-26T20:36:20.268ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = b82a4171-23b2-444b-9597-09085a7a2fb8 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
44622024-08-26T20:36:20.268ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
44632024-08-26T20:36:20.268ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
44642024-08-26T20:36:20.268ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:454 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
44652024-08-26T20:36:20.268ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 17.861376ms iteration = 17 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
44662024-08-26T20:36:20.269ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
44672024-08-26T20:36:20.269ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
44682024-08-26T20:36:20.269ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
44692024-08-26T20:36:20.269ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
44702024-08-26T20:36:20.270ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
44712024-08-26T20:36:20.270ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
44722024-08-26T20:36:20.270ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
44732024-08-26T20:36:20.270ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
44742024-08-26T20:36:20.271ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
44752024-08-26T20:36:20.272ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
44762024-08-26T20:36:20.272ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
44772024-08-26T20:36:20.272ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
44782024-08-26T20:36:20.272ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
44792024-08-26T20:36:20.272ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
44802024-08-26T20:36:20.272ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
44812024-08-26T20:36:20.272ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
44822024-08-26T20:36:20.272ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
44832024-08-26T20:36:20.272ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
44842024-08-26T20:36:20.273ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
44852024-08-26T20:36:20.273ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
44862024-08-26T20:36:20.273ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
44872024-08-26T20:36:20.273ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
44882024-08-26T20:36:20.274ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = d840a78c-2f0d-4cdf-9962-31a3dae384a2 dns_zone_name = oxide.test found_so_far = 1 version = 2
44892024-08-26T20:36:20.274ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
44902024-08-26T20:36:20.274ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-08-26 20:29:01.487349 UTC dns_group = external generation = 2
44912024-08-26T20:36:20.274ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 23.914683ms iteration = 9 status = {"generation":2}
44922024-08-26T20:36:20.274ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
44932024-08-26T20:36:20.274ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
44942024-08-26T20:36:20.274ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
44952024-08-26T20:36:20.274ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
44962024-08-26T20:36:20.274ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
44972024-08-26T20:36:20.275ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = b82a4171-23b2-444b-9597-09085a7a2fb8 dns_zone_name = control-plane.oxide.internal found_so_far = 50 version = 1
44982024-08-26T20:36:20.275ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
44992024-08-26T20:36:20.275ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-08-26 20:29:01.089496 UTC dns_group = internal generation = 1
45002024-08-26T20:36:20.275ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 24.819783ms iteration = 9 status = {"generation":1}
45012024-08-26T20:36:20.275ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
45022024-08-26T20:36:20.275ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
45032024-08-26T20:36:20.276ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
45042024-08-26T20:36:20.276ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 25.929652ms iteration = 49 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
45052024-08-26T20:36:20.277ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
45062024-08-26T20:36:20.277ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
45072024-08-26T20:36:20.277ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
45082024-08-26T20:36:20.278ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: c9d3ee27-fe3f-422b-93ce-695a8356d9d2, lookup_type: ById(c9d3ee27-fe3f-422b-93ce-695a8356d9d2) } result = Ok(())
45092024-08-26T20:36:20.278ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
45102024-08-26T20:36:20.278ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
45112024-08-26T20:36:20.279ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 28.721759ms iteration = 17 status = 0
45122024-08-26T20:36:20.283ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-08-26T20:28:18.089116Z, time_modified: 2024-08-26T20:28:18.089116Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 80a72585-f38a-4ccb-b69a-caafab415ca2, db_silo: Silo { identity: SiloIdentity { id: 80a72585-f38a-4ccb-b69a-caafab415ca2, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-08-26T20:29:01.450089Z, time_modified: 2024-08-26T20:29:01.450089Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(4)) }, tls_certs: [TlsCertificate { digest: "81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 80a72585-f38a-4ccb-b69a-caafab415ca2, db_silo: Silo { identity: SiloIdentity { id: 80a72585-f38a-4ccb-b69a-caafab415ca2, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-08-26T20:29:01.450089Z, time_modified: 2024-08-26T20:29:01.450089Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(4)) }, tls_certs: [TlsCertificate { digest: "81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923" }] }) }
45132024-08-26T20:36:20.283ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 32.964318ms iteration = 9 status = {"by_dns_name":{"default-silo.sys.oxide.test":{"silo_id":"001de000-5110-4000-8000-000000000000","tls_certs":[]},"recovery.sys.oxide.test":{"silo_id":"80a72585-f38a-4ccb-b69a-caafab415ca2","tls_certs":["81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923"]}},"default_endpoint":{"silo_id":"80a72585-f38a-4ccb-b69a-caafab415ca2","tls_certs":["81025e3575ef76091cf2c28463b021e7ae7d3f9402249731b54bf05a551af923"]},"warnings":["silo 001de000-5110-4000-8000-000000000000 with DNS name \\"default-silo.sys.oxide.test\\" has no usable certificates"]}
45142024-08-26T20:36:20.291ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 41.021694ms iteration = 18 status = {"physical_disks_added":0}
45152024-08-26T20:36:20.422ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
45162024-08-26T20:36:20.458ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
45172024-08-26T20:36:20.458ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
45182024-08-26T20:36:20.458ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "c8d35ba1-610c-4ff8-9276-062c7c14932d", "content-length": "26", "date": "Mon, 26 Aug 2024 20:36:20 GMT"} })
45192024-08-26T20:36:20.458ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
45202024-08-26T20:36:20.460ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
45212024-08-26T20:36:20.604ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1058 zone_address = fd00:1122:3344:101::2
45222024-08-26T20:36:20.604ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
45232024-08-26T20:36:20.604ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "b2a1beda-4221-4ee7-bd73-fd3c41eda1bc", "content-length": "26", "date": "Mon, 26 Aug 2024 20:36:20 GMT"} })
45242024-08-26T20:36:20.604ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1063 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
45252024-08-26T20:36:20.606ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
45262024-08-26T20:36:20.619ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
45272024-08-26T20:36:20.619ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::8)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "fdd63d93-bb5c-4e41-a997-f3fa0dcd4fff", "date": "Mon, 26 Aug 2024 20:36:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
45282024-08-26T20:36:20.619ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
45292024-08-26T20:36:20.620ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::9)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "2d822206-30dd-4b27-8ebe-6a9d47bd4193", "date": "Mon, 26 Aug 2024 20:36:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
45302024-08-26T20:36:20.620ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
45312024-08-26T20:36:20.622ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
45322024-08-26T20:36:20.622ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 372.322882ms iteration = 11 status = {"generation":2,"server_results":{"[fd00:1122:3344:101::8]:5353":{"Ok":null},"[fd00:1122:3344:101::9]:5353":{"Ok":null}}}
45332024-08-26T20:36:20.626ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
45342024-08-26T20:36:20.626ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = v2p_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/v2p", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "7d532bda-6e7c-455b-94a8-1b28687c2c0a", "content-length": "691", "date": "Mon, 26 Aug 2024 20:36:19 GMT"} })
45352024-08-26T20:36:20.626ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = sock
45362024-08-26T20:36:20.626ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = sock
45372024-08-26T20:36:20.626ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = sock
45382024-08-26T20:36:20.628ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 377.785723ms iteration = 17 status = {}
45392024-08-26T20:36:20.654ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
45402024-08-26T20:36:20.760ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 509.696567ms iteration = 17 status = {}
45412024-08-26T20:36:20.772ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1058 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 zone_address = fd00:1122:3344:101::2
45422024-08-26T20:36:20.772ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
45432024-08-26T20:36:20.773ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12225), path: "/local/switch-id", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "1a08ca8d-5fd8-4d53-97c1-f7cedc13a798", "content-length": "26", "date": "Mon, 26 Aug 2024 20:36:20 GMT"} })
45442024-08-26T20:36:20.773ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1063 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
45452024-08-26T20:36:20.775ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1095 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
45462024-08-26T20:36:20.783ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
45472024-08-26T20:36:20.783ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "a672e1aa-15dc-425e-82d7-5a10675de00a", "content-length": "372", "date": "Mon, 26 Aug 2024 20:36:20 GMT"} })
45482024-08-26T20:36:20.783ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
45492024-08-26T20:36:20.783ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
45502024-08-26T20:36:20.785ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
45512024-08-26T20:36:20.785ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
45522024-08-26T20:36:20.787ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
45532024-08-26T20:36:20.787ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
45542024-08-26T20:36:20.788ZDEBGnexus (BackgroundTasks): client request SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
45552024-08-26T20:36:20.789ZDEBGnexus (BackgroundTasks): client response SledAgent = c9d3ee27-fe3f-422b-93ce-695a8356d9d2 background_task = vpc_route_manager result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/vpc-routes", query: None, fragment: None }, status: 204, headers: {"x-request-id": "76bb14fd-8a83-410a-932d-5693dae5db54", "date": "Mon, 26 Aug 2024 20:36:20 GMT"} })
45562024-08-26T20:36:20.790ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 540.270632ms iteration = 17 status = {}
45572024-08-26T20:36:20.866ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
45582024-08-26T20:36:20.866ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/rpw/nat/ipv4/gen", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "c820a40e-2e01-45f5-935c-33cf354e59f1", "content-length": "2", "date": "Mon, 26 Aug 2024 20:36:20 GMT"} })
45592024-08-26T20:36:20.868ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
45602024-08-26T20:36:20.869ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
45612024-08-26T20:36:20.870ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 619.880948ms iteration = 17 status = null
45622024-08-26T20:36:20.876ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
45632024-08-26T20:36:20.915ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
45642024-08-26T20:36:20.915ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:3::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "e64ccf63-4f49-49ab-b9cf-b2194ca601a6", "date": "Mon, 26 Aug 2024 20:36:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
45652024-08-26T20:36:20.915ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
45662024-08-26T20:36:20.916ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:2::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "15027fdc-0f1f-4978-bfcf-09c90d8208e2", "date": "Mon, 26 Aug 2024 20:36:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
45672024-08-26T20:36:20.916ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
45682024-08-26T20:36:20.918ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:1::1)), port: Some(5353), path: "/config", query: None, fragment: None }, status: 204, headers: {"x-request-id": "ec26836b-14ee-4e3b-9d51-7e1337a2ed33", "date": "Mon, 26 Aug 2024 20:36:20 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
45692024-08-26T20:36:20.918ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
45702024-08-26T20:36:20.921ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353, [fd00:1122:3344:3::1]:5353] }
45712024-08-26T20:36:20.921ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 670.861254ms iteration = 11 status = {"generation":1,"server_results":{"[fd00:1122:3344:1::1]:5353":{"Ok":null},"[fd00:1122:3344:2::1]:5353":{"Ok":null},"[fd00:1122:3344:3::1]:5353":{"Ok":null}}}
45722024-08-26T20:36:20.936ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
45732024-08-26T20:36:20.936ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
45742024-08-26T20:36:20.938ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:147 port = SwitchPort { id: 77948b01-64a9-403e-be7b-4ef5e426dda4, rack_id: 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(735476d6-05ca-4fbd-a182-ab082f700355) } rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
45752024-08-26T20:36:20.938ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
45762024-08-26T20:36:20.938ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
45772024-08-26T20:36:20.945ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
45782024-08-26T20:36:20.945ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/port/qsfp0/settings", query: Some("tag=omicron"), fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "2e6996de-dfcd-47a0-a122-fc2cb29ad81c", "content-length": "121", "date": "Mon, 26 Aug 2024 20:36:20 GMT"} })
45792024-08-26T20:36:20.945ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1714 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.2.174], params: LinkCreate { autoneg: false, fec: None, kr: false, lane: Some(LinkId(0)), speed: Speed40G } }} } switch_port_id = "qsfp0"
45802024-08-26T20:36:20.945ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1741 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = 735476d6-05ca-4fbd-a182-ab082f700355
45812024-08-26T20:36:20.945ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:379 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
45822024-08-26T20:36:20.945ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {Switch0: {(10.151.2.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
45832024-08-26T20:36:20.945ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:392 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
45842024-08-26T20:36:20.945ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:398 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 routes = {}
45852024-08-26T20:36:20.945ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:418 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29
45862024-08-26T20:36:20.945ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
45872024-08-26T20:36:20.945ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
45882024-08-26T20:36:20.947ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
45892024-08-26T20:36:20.948ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv4", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "926014e5-7a5a-451e-aa08-ef2c358ef92c", "content-length": "2", "date": "Mon, 26 Aug 2024 20:36:20 GMT"} })
45902024-08-26T20:36:20.948ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
45912024-08-26T20:36:20.948ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::2)), port: Some(12224), path: "/loopback/ipv6", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "d2cb9d85-5344-45ed-a6e9-b1fa1f9d91e8", "content-length": "52", "date": "Mon, 26 Aug 2024 20:36:20 GMT"} })
45922024-08-26T20:36:20.948ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0" }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:467 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch_location = Switch0
45932024-08-26T20:36:20.948ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
45942024-08-26T20:36:21.036ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/switch-ports", query: None, fragment: None }, status: 204, headers: {"x-request-id": "b0fb4c35-5428-4b2e-b699-bd0901c654cf", "date": "Mon, 26 Aug 2024 20:36:20 GMT"} })
45952024-08-26T20:36:21.036ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:770 rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 switch = Switch0
45962024-08-26T20:36:21.036ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
45972024-08-26T20:36:21.040ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 7cfa2cd8-5cab-4eab-9ee4-737101ca1e29 result = Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::1)), port: Some(12345), path: "/network-bootstore-config", query: None, fragment: None }, status: 200, headers: {"content-type": "application/json", "x-request-id": "4cf46442-e883-4655-b7d2-a07142d63b3b", "content-length": "667", "date": "Mon, 26 Aug 2024 20:36:20 GMT"} })
45982024-08-26T20:36:21.040ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
45992024-08-26T20:36:21.040ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
46002024-08-26T20:36:21.043ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
46012024-08-26T20:36:21.043ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
46022024-08-26T20:36:21.047ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
46032024-08-26T20:36:21.047ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
46042024-08-26T20:36:21.048ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
46052024-08-26T20:36:21.048ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
46062024-08-26T20:36:21.054ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 803.990649ms iteration = 17 status = {}
46072024-08-26T20:36:21.096ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 308 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 2bcd7735-142c-4b5d-b744-261e5e997631 response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5
46082024-08-26T20:36:30.249ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 50 reason = Timeout
46092024-08-26T20:36:30.250ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
46102024-08-26T20:36:30.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
46112024-08-26T20:36:30.250ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 50 reason = Timeout
46122024-08-26T20:36:30.250ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
46132024-08-26T20:36:30.250ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
46142024-08-26T20:36:30.251ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
46152024-08-26T20:36:30.251ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
46162024-08-26T20:36:30.252ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
46172024-08-26T20:36:30.253ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 2.775486ms iteration = 50 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
46182024-08-26T20:36:30.254ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
46192024-08-26T20:36:30.254ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
46202024-08-26T20:36:30.254ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
46212024-08-26T20:36:30.255ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
46222024-08-26T20:36:30.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
46232024-08-26T20:36:30.257ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
46242024-08-26T20:36:30.259ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 6588063f-ab6d-40dd-8d34-b030db9d7460, lookup_type: ById(6588063f-ab6d-40dd-8d34-b030db9d7460) } result = Ok(())
46252024-08-26T20:36:30.259ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
46262024-08-26T20:36:30.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
46272024-08-26T20:36:30.265ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 original_time_created = 2024-08-26 20:28:12.023487 UTC target_id = 6588063f-ab6d-40dd-8d34-b030db9d7460 time_created = 2024-08-26 20:28:12.023487 UTC
46282024-08-26T20:36:30.265ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 15.747497ms iteration = 50 status = {"status":"target blueprint unchanged","target_id":"6588063f-ab6d-40dd-8d34-b030db9d7460","time_created":"2024-08-26T20:28:12.023487Z"}
46292024-08-26T20:36:31.096ZINFOnexus (dropshot): request completed file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914 latency_us = 345 local_addr = [fd00:1122:3344:101::c]:56857 method = GET remote_addr = [fd00:1122:3344:101::d]:62482 req_id = 715cf206-551d-42b2-a2db-db77ffc2f9de response_code = 200 uri = /ac8efda2-bd6c-4573-a0bc-ffd21bb4d8a5