Max level shown:
1 [ Nov 21 19:33:05 Enabled. ]
2 [ Nov 21 19:33:05 Rereading configuration. ]
3 [ Nov 21 19:33:06 Rereading configuration. ]
4 [ Nov 21 19:33:13 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/omicron-nexus/bin/nexus /var/svc/manifest/site/nexus/config.toml &"). ]
5 [ Nov 21 19:33:13 Method "start" exited with status 0. ]
6 note: configured to log to "/dev/stdout"
72024-11-21T19:33:13.341ZDEBGnexus: registered DTrace probes
82024-11-21T19:33:13.343ZINFOnexus: setting up nexus server file = nexus/src/lib.rs:75
92024-11-21T19:33:13.352ZINFOnexus (ServerContext): registering Oso class class = Action file = nexus/auth/src/authz/oso_generic.rs:68
102024-11-21T19:33:13.352ZINFOnexus (ServerContext): registering Oso class class = AnyActor file = nexus/auth/src/authz/oso_generic.rs:68
112024-11-21T19:33:13.352ZINFOnexus (ServerContext): registering Oso class class = AuthenticatedActor file = nexus/auth/src/authz/oso_generic.rs:68
122024-11-21T19:33:13.352ZINFOnexus (ServerContext): registering Oso class class = BlueprintConfig file = nexus/auth/src/authz/oso_generic.rs:68
132024-11-21T19:33:13.352ZINFOnexus (ServerContext): registering Oso class class = Database file = nexus/auth/src/authz/oso_generic.rs:68
142024-11-21T19:33:13.352ZINFOnexus (ServerContext): registering Oso class class = DnsConfig file = nexus/auth/src/authz/oso_generic.rs:68
152024-11-21T19:33:13.352ZINFOnexus (ServerContext): registering Oso class class = Fleet file = nexus/auth/src/authz/oso_generic.rs:68
162024-11-21T19:33:13.352ZINFOnexus (ServerContext): registering Oso class class = Inventory file = nexus/auth/src/authz/oso_generic.rs:68
172024-11-21T19:33:13.352ZINFOnexus (ServerContext): registering Oso class class = IpPoolList file = nexus/auth/src/authz/oso_generic.rs:68
182024-11-21T19:33:13.352ZINFOnexus (ServerContext): registering Oso class class = ConsoleSessionList file = nexus/auth/src/authz/oso_generic.rs:68
192024-11-21T19:33:13.352ZINFOnexus (ServerContext): registering Oso class class = DeviceAuthRequestList file = nexus/auth/src/authz/oso_generic.rs:68
202024-11-21T19:33:13.352ZINFOnexus (ServerContext): registering Oso class class = SiloCertificateList file = nexus/auth/src/authz/oso_generic.rs:68
212024-11-21T19:33:13.352ZINFOnexus (ServerContext): registering Oso class class = SiloIdentityProviderList file = nexus/auth/src/authz/oso_generic.rs:68
222024-11-21T19:33:13.352ZINFOnexus (ServerContext): registering Oso class class = SiloUserList file = nexus/auth/src/authz/oso_generic.rs:68
232024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = Project file = nexus/auth/src/authz/oso_generic.rs:68
242024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = Disk file = nexus/auth/src/authz/oso_generic.rs:68
252024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = Snapshot file = nexus/auth/src/authz/oso_generic.rs:68
262024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = ProjectImage file = nexus/auth/src/authz/oso_generic.rs:68
272024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = Instance file = nexus/auth/src/authz/oso_generic.rs:68
282024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = IpPool file = nexus/auth/src/authz/oso_generic.rs:68
292024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = InstanceNetworkInterface file = nexus/auth/src/authz/oso_generic.rs:68
302024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = Vpc file = nexus/auth/src/authz/oso_generic.rs:68
312024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = VpcRouter file = nexus/auth/src/authz/oso_generic.rs:68
322024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = InternetGateway file = nexus/auth/src/authz/oso_generic.rs:68
332024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = InternetGatewayIpPool file = nexus/auth/src/authz/oso_generic.rs:68
342024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = InternetGatewayIpAddress file = nexus/auth/src/authz/oso_generic.rs:68
352024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = RouterRoute file = nexus/auth/src/authz/oso_generic.rs:68
362024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = VpcSubnet file = nexus/auth/src/authz/oso_generic.rs:68
372024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = FloatingIp file = nexus/auth/src/authz/oso_generic.rs:68
382024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = Image file = nexus/auth/src/authz/oso_generic.rs:68
392024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = SiloImage file = nexus/auth/src/authz/oso_generic.rs:68
402024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = AddressLot file = nexus/auth/src/authz/oso_generic.rs:68
412024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = Blueprint file = nexus/auth/src/authz/oso_generic.rs:68
422024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = LoopbackAddress file = nexus/auth/src/authz/oso_generic.rs:68
432024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = Certificate file = nexus/auth/src/authz/oso_generic.rs:68
442024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = ConsoleSession file = nexus/auth/src/authz/oso_generic.rs:68
452024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = DeviceAuthRequest file = nexus/auth/src/authz/oso_generic.rs:68
462024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = DeviceAccessToken file = nexus/auth/src/authz/oso_generic.rs:68
472024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = PhysicalDisk file = nexus/auth/src/authz/oso_generic.rs:68
482024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = Rack file = nexus/auth/src/authz/oso_generic.rs:68
492024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = RoleBuiltin file = nexus/auth/src/authz/oso_generic.rs:68
502024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = SshKey file = nexus/auth/src/authz/oso_generic.rs:68
512024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = Silo file = nexus/auth/src/authz/oso_generic.rs:68
522024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = SiloUser file = nexus/auth/src/authz/oso_generic.rs:68
532024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = SiloGroup file = nexus/auth/src/authz/oso_generic.rs:68
542024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = IdentityProvider file = nexus/auth/src/authz/oso_generic.rs:68
552024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = SamlIdentityProvider file = nexus/auth/src/authz/oso_generic.rs:68
562024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = Sled file = nexus/auth/src/authz/oso_generic.rs:68
572024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = TufRepo file = nexus/auth/src/authz/oso_generic.rs:68
582024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = TufArtifact file = nexus/auth/src/authz/oso_generic.rs:68
592024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = Zpool file = nexus/auth/src/authz/oso_generic.rs:68
602024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = Service file = nexus/auth/src/authz/oso_generic.rs:68
612024-11-21T19:33:13.353ZINFOnexus (ServerContext): registering Oso class class = UserBuiltin file = nexus/auth/src/authz/oso_generic.rs:68
622024-11-21T19:33:13.353ZINFOnexus (ServerContext): full Oso configuration config = #\n# Oso configuration for Omicron\n# This file is augmented by generated snippets.\n#\n\n#\n# ACTOR TYPES AND BASIC RULES\n#\n\n# `AnyActor` includes both authenticated and unauthenticated users.\nactor AnyActor {}\n\n# An `AuthenticatedActor` has an identity in the system. All of our operations\n# today require that an actor be authenticated.\nactor AuthenticatedActor {}\n\n# For any resource, `actor` can perform action `action` on it if they're\n# authenticated and their role(s) give them the corresponding permission on that\n# resource.\nallow(actor: AnyActor, action: Action, resource) if\n actor.authenticated and\n has_permission(actor.authn_actor.unwrap(), action.to_perm(), resource);\n\n# Define role relationships\nhas_role(actor: AuthenticatedActor, role: String, resource: Resource)\n\tif resource.has_role(actor, role);\n\n#\n# ROLES AND PERMISSIONS IN THE FLEET/SILO/PROJECT HIERARCHY\n#\n# We define the following permissions for most resources in the system:\n#\n# - "create_child": required to create child resources (of any type)\n#\n# - "list_children": required to list child resources (of all types) of a\n# resource\n#\n# - "modify": required to modify or delete a resource\n#\n# - "read": required to read a resource\n#\n# We define the following predefined roles for only a few high-level resources:\n# the Fleet (see below), Silo, Organization, and Project. The specific roles\n# are oriented around intended use-cases:\n#\n# - "admin": has all permissions on the resource\n#\n# - "collaborator": has "read", "list_children", and "create_child", plus\n# the "admin" role for child resources. The idea is that if you're an\n# Organization Collaborator, you have full control over the Projects within\n# the Organization, but you cannot modify or delete the Organization itself.\n#\n# - "viewer": has "read" and "list_children" on a resource\n#\n# Below the Project level, permissions are granted via roles at the Project\n# level. For example, for someone to be able to create, modify, or delete any\n# Instances, they must be granted project.collaborator, which means they can\n# create, modify, or delete _all_ resources in the Project.\n#\n# The complete set of predefined roles:\n#\n# - fleet.admin (superuser for the whole system)\n# - fleet.collaborator (can manage Silos)\n# - fleet.viewer (can read most non-siloed resources in the system)\n# - silo.admin (superuser for the silo)\n# - silo.collaborator (can create and own Organizations)\n# - silo.viewer (can read most resources within the Silo)\n# - organization.admin (complete control over an organization)\n# - organization.collaborator (can manage Projects)\n# - organization.viewer (can read most resources within the Organization)\n# - project.admin (complete control over a Project)\n# - project.collaborator (can manage all resources within the Project)\n# - project.viewer (can read most resources within the Project)\n#\n# Outside the Silo/Organization/Project hierarchy, we (currently) treat most\n# resources as nested under Fleet or else a synthetic resource (see below). We\n# do not yet support role assignments on anything other than Fleet, Silo,\n# Organization, or Project.\n#\n\n# "Fleet" is a global singleton representing the whole system. The name comes\n# from the idea described in RFD 24, but it's not quite right. This probably\n# should be more like "Region" or "AvailabilityZone". The precise boundaries\n# have not yet been figured out.\nresource Fleet {\n\tpermissions = [\n\t "list_children",\n\t "modify",\n\t "read",\n\t "create_child",\n\t];\n\n\troles = [\n\t # Roles that can be attached by users\n\t "admin",\n\t "collaborator",\n\t "viewer",\n\n\t # Internal-only roles\n\t "external-authenticator"\n\t];\n\n\t# Roles implied by other roles on this resource\n\t"viewer" if "collaborator";\n\t"collaborator" if "admin";\n\n\t# Permissions granted directly by roles on this resource\n\t"list_children" if "viewer";\n\t"read" if "viewer";\n\t"create_child" if "collaborator";\n\t"modify" if "admin";\n}\n\n# For fleets specifically, roles can be conferred by roles on the user's Silo.\nhas_role(actor: AuthenticatedActor, role: String, _: Fleet) if\n\tsilo_role in actor.confers_fleet_role(role) and\n\thas_role(actor, silo_role, actor.silo.unwrap());\n\nresource Silo {\n\tpermissions = [\n\t "list_children",\n\t "modify",\n\t "read",\n\t "create_child",\n\t];\n\troles = [ "admin", "collaborator", "viewer" ];\n\n\t# Roles implied by other roles on this resource\n\t"viewer" if "collaborator";\n\t"collaborator" if "admin";\n\n\t# Permissions granted directly by roles on this resource\n\t"list_children" if "viewer";\n\t"read" if "viewer";\n\n\t"create_child" if "collaborator";\n\t"modify" if "admin";\n\n\t# Permissions implied by roles on this resource's parent (Fleet). Fleet\n\t# privileges allow a user to see and potentially administer the Silo,\n\t# but they do not give anyone permission to look at anything inside the\n\t# Silo. To achieve this, we use permission rules here. (If we granted\n\t# Fleet administrators _roles_ on the Silo, then those would cascade\n\t# into the Silo as well.)\n\trelations = { parent_fleet: Fleet };\n\t"read" if "viewer" on "parent_fleet";\n\t"modify" if "collaborator" on "parent_fleet";\n\n\t# external authenticator has to create silo users\n\t"list_children" if "external-authenticator" on "parent_fleet";\n\t"create_child" if "external-authenticator" on "parent_fleet";\n}\n\nhas_relation(fleet: Fleet, "parent_fleet", silo: Silo)\n\tif silo.fleet = fleet;\n\n# As a special case, all authenticated users can read their own Silo. That's\n# not quite the same as having the "viewer" role. For example, they cannot list\n# Organizations in the Silo.\n#\n# One reason this is necessary is because if an unprivileged user tries to\n# create an Organization using "POST /organizations", they should get back a 403\n# (which implies they're able to see /organizations, which is essentially seeing\n# the Silo itself) rather than a 404. This behavior isn't a hard constraint\n# (i.e., you could reasonably get a 404 for an API you're not allowed to call).\n# Nor is the implementation (i.e., we could special-case this endpoint somehow).\n# But granting this permission is the simplest way to keep this endpoint's\n# behavior consistent with the rest of the API.\n#\n# This rule is also used to determine if a user can list the identity providers\n# in the Silo (which they should be able to), since that's predicated on being\n# able to read the Silo.\n#\n# It's unclear what else would break if users couldn't see their own Silo.\nhas_permission(actor: AuthenticatedActor, "read", silo: Silo)\n\tif silo in actor.silo;\n\nresource Project {\n\tpermissions = [\n\t "list_children",\n\t "modify",\n\t "read",\n\t "create_child",\n\t];\n\troles = [ "admin", "collaborator", "viewer" ];\n\n\t# Roles implied by other roles on this resource\n\t"viewer" if "collaborator";\n\t"collaborator" if "admin";\n\n\t# Permissions granted directly by roles on this resource\n\t"list_children" if "viewer";\n\t"read" if "viewer";\n\t"create_child" if "collaborator";\n\t"modify" if "admin";\n\n\t# Roles implied by roles on this resource's parent (Silo)\n\trelations = { parent_silo: Silo };\n\t"admin" if "collaborator" on "parent_silo";\n\t"viewer" if "viewer" on "parent_silo";\n}\nhas_relation(silo: Silo, "parent_silo", project: Project)\n\tif project.silo = silo;\n\n#\n# GENERAL RESOURCES OUTSIDE THE SILO/PROJECT HIERARCHY\n#\n# Many resources use snippets of Polar generated by the `authz_resource!` Rust\n# macro. Some resources require custom Polar code. Those appear here.\n#\n\nresource Certificate {\n\tpermissions = [ "read", "modify" ];\n\trelations = { parent_silo: Silo, parent_fleet: Fleet };\n\n\t# Fleet-level and silo-level roles both grant privileges on certificates.\n\t"read" if "admin" on "parent_silo";\n\t"modify" if "admin" on "parent_silo";\n\t"read" if "admin" on "parent_fleet";\n\t"modify" if "admin" on "parent_fleet";\n}\nhas_relation(silo: Silo, "parent_silo", certificate: Certificate)\n\tif certificate.silo = silo;\nhas_relation(fleet: Fleet, "parent_fleet", certificate: Certificate)\n\tif certificate.silo.fleet = fleet;\n\nresource SiloUser {\n\tpermissions = [\n\t "list_children",\n\t "modify",\n\t "read",\n\t "create_child",\n\t];\n\n\t# Fleet and Silo administrators can manage a Silo's users. This is one\n\t# of the only areas of Silo configuration that Fleet Administrators have\n\t# permissions on.\n\trelations = { parent_silo: Silo, parent_fleet: Fleet };\n\t"list_children" if "read" on "parent_silo";\n\t"read" if "read" on "parent_silo";\n\t"modify" if "admin" on "parent_silo";\n\t"create_child" if "admin" on "parent_silo";\n\t"list_children" if "read" on "parent_fleet";\n\t"read" if "read" on "parent_fleet";\n\t"modify" if "admin" on "parent_fleet";\n\t"create_child" if "admin" on "parent_fleet";\n}\nhas_relation(silo: Silo, "parent_silo", user: SiloUser)\n\tif user.silo = silo;\nhas_relation(fleet: Fleet, "parent_fleet", user: SiloUser)\n\tif user.silo.fleet = fleet;\n\n# authenticated actors have all permissions on themselves\nhas_permission(actor: AuthenticatedActor, _perm: String, silo_user: SiloUser)\n if actor.equals_silo_user(silo_user);\n\nhas_permission(actor: AuthenticatedActor, "read", silo_user: SiloUser)\n if silo_user.silo in actor.silo;\n\nresource SiloGroup {\n\tpermissions = [\n\t "list_children",\n\t "modify",\n\t "read",\n\t "create_child",\n\t];\n\n\trelations = { parent_silo: Silo };\n\t"list_children" if "read" on "parent_silo";\n\t"read" if "read" on "parent_silo";\n\t"modify" if "admin" on "parent_silo";\n\t"create_child" if "admin" on "parent_silo";\n}\nhas_relation(silo: Silo, "parent_silo", group: SiloGroup)\n\tif group.silo = silo;\n\nresource SshKey {\n\tpermissions = [ "read", "modify" ];\n\trelations = { silo_user: SiloUser };\n\n\t"read" if "read" on "silo_user";\n\t"modify" if "modify" on "silo_user";\n}\nhas_relation(user: SiloUser, "silo_user", ssh_key: SshKey)\n\tif ssh_key.silo_user = user;\n\nresource IdentityProvider {\n\tpermissions = [\n\t "read",\n\t "modify",\n\t "create_child",\n\t "list_children",\n\t];\n\trelations = { parent_silo: Silo, parent_fleet: Fleet };\n\n\t# Silo-level roles grant privileges on identity providers.\n\t"read" if "viewer" on "parent_silo";\n\t"list_children" if "viewer" on "parent_silo";\n\t"modify" if "admin" on "parent_silo";\n\t"create_child" if "admin" on "parent_silo";\n\n\t# Fleet-level roles also grant privileges on identity providers.\n\t"read" if "viewer" on "parent_fleet";\n\t"list_children" if "viewer" on "parent_fleet";\n\t"modify" if "admin" on "parent_fleet";\n\t"create_child" if "admin" on "parent_fleet";\n}\nhas_relation(silo: Silo, "parent_silo", identity_provider: IdentityProvider)\n\tif identity_provider.silo = silo;\nhas_relation(fleet: Fleet, "parent_fleet", collection: IdentityProvider)\n\tif collection.silo.fleet = fleet;\n\nresource SamlIdentityProvider {\n\tpermissions = [\n\t "read",\n\t "modify",\n\t "create_child",\n\t "list_children",\n\t];\n\trelations = { parent_silo: Silo, parent_fleet: Fleet };\n\n\t# Silo-level roles grant privileges on identity providers.\n\t"read" if "viewer" on "parent_silo";\n\t"list_children" if "viewer" on "parent_silo";\n\t"modify" if "admin" on "parent_silo";\n\t"create_child" if "admin" on "parent_silo";\n\n\t# Fleet-level roles also grant privileges on identity providers.\n\t"read" if "viewer" on "parent_fleet";\n\t"list_children" if "viewer" on "parent_fleet";\n\t"modify" if "admin" on "parent_fleet";\n\t"create_child" if "admin" on "parent_fleet";\n}\nhas_relation(silo: Silo, "parent_silo", saml_identity_provider: SamlIdentityProvider)\n\tif saml_identity_provider.silo = silo;\nhas_relation(fleet: Fleet, "parent_fleet", collection: SamlIdentityProvider)\n\tif collection.silo.fleet = fleet;\n\n#\n# SYNTHETIC RESOURCES OUTSIDE THE SILO HIERARCHY\n#\n# The resources here do not correspond to anything that appears explicitly in\n# the API or is stored in the database. These are used either at the top level\n# of the API path (e.g., "/v1/system/ip-pools") or as an implementation detail of the system\n# (in the case of console sessions and "Database"). The policies are\n# either statically-defined in this file or driven by role assignments on the\n# Fleet. None of these resources defines their own roles.\n#\n\n# Describes the policy for reading and modifying DNS configuration\n# (both internal and external)\nresource DnsConfig {\n\tpermissions = [ "read", "modify" ];\n\trelations = { parent_fleet: Fleet };\n\t# "external-authenticator" requires these permissions because that's the\n\t# context that Nexus uses when creating and deleting Silos. These\n\t# operations necessarily need to read and modify DNS configuration.\n\t"read" if "external-authenticator" on "parent_fleet";\n\t"modify" if "external-authenticator" on "parent_fleet";\n\t# "admin" on the parent fleet also gets these permissions, primarily for\n\t# the test suite.\n\t"read" if "admin" on "parent_fleet";\n\t"modify" if "admin" on "parent_fleet";\n}\nhas_relation(fleet: Fleet, "parent_fleet", dns_config: DnsConfig)\n\tif dns_config.fleet = fleet;\n\n# Describes the policy for accessing blueprints\nresource BlueprintConfig {\n\tpermissions = [\n\t "list_children", # list blueprints\n\t "create_child", # create blueprint\n\t "read", # read the current target\n\t "modify", # change the current target\n\t];\n\n\trelations = { parent_fleet: Fleet };\n\t"create_child" if "admin" on "parent_fleet";\n\t"modify" if "admin" on "parent_fleet";\n\t"list_children" if "viewer" on "parent_fleet";\n\t"read" if "viewer" on "parent_fleet";\n}\nhas_relation(fleet: Fleet, "parent_fleet", list: BlueprintConfig)\n\tif list.fleet = fleet;\n\n# Describes the policy for reading and modifying low-level inventory\nresource Inventory {\n\tpermissions = [ "read", "modify" ];\n\trelations = { parent_fleet: Fleet };\n\t"read" if "viewer" on "parent_fleet";\n\t"modify" if "admin" on "parent_fleet";\n}\nhas_relation(fleet: Fleet, "parent_fleet", inventory: Inventory)\n\tif inventory.fleet = fleet;\n\n# Describes the policy for accessing "/v1/system/ip-pools" in the API\nresource IpPoolList {\n\tpermissions = [\n\t "list_children",\n\t "modify",\n\t "create_child",\n\t];\n\n\t# Fleet Administrators can create or modify the IP Pools list.\n\trelations = { parent_fleet: Fleet };\n\t"modify" if "admin" on "parent_fleet";\n\t"create_child" if "admin" on "parent_fleet";\n\n\t# Fleet Viewers can list IP Pools\n\t"list_children" if "viewer" on "parent_fleet";\n}\nhas_relation(fleet: Fleet, "parent_fleet", ip_pool_list: IpPoolList)\n\tif ip_pool_list.fleet = fleet;\n\n# Any authenticated user can create a child of a provided IP Pool.\n# This is necessary to use the pools when provisioning instances.\nhas_permission(actor: AuthenticatedActor, "create_child", ip_pool: IpPool)\n\tif silo in actor.silo and silo.fleet = ip_pool.fleet;\n\n# Describes the policy for creating and managing web console sessions.\nresource ConsoleSessionList {\n\tpermissions = [ "create_child" ];\n\trelations = { parent_fleet: Fleet };\n\t"create_child" if "external-authenticator" on "parent_fleet";\n}\nhas_relation(fleet: Fleet, "parent_fleet", collection: ConsoleSessionList)\n\tif collection.fleet = fleet;\n\n# Describes the policy for creating and managing device authorization requests.\nresource DeviceAuthRequestList {\n\tpermissions = [ "create_child" ];\n\trelations = { parent_fleet: Fleet };\n\t"create_child" if "external-authenticator" on "parent_fleet";\n}\nhas_relation(fleet: Fleet, "parent_fleet", collection: DeviceAuthRequestList)\n\tif collection.fleet = fleet;\n\n# Describes the policy for creating and managing Silo certificates\nresource SiloCertificateList {\n\tpermissions = [ "list_children", "create_child" ];\n\n\trelations = { parent_silo: Silo, parent_fleet: Fleet };\n\n\t# Both Fleet and Silo administrators can see and modify the Silo's\n\t# certificates.\n\t"list_children" if "admin" on "parent_silo";\n\t"list_children" if "admin" on "parent_fleet";\n\t"create_child" if "admin" on "parent_silo";\n\t"create_child" if "admin" on "parent_fleet";\n}\nhas_relation(silo: Silo, "parent_silo", collection: SiloCertificateList)\n\tif collection.silo = silo;\nhas_relation(fleet: Fleet, "parent_fleet", collection: SiloCertificateList)\n\tif collection.silo.fleet = fleet;\n\n# Describes the policy for creating and managing Silo identity providers\nresource SiloIdentityProviderList {\n\tpermissions = [ "list_children", "create_child" ];\n\n\trelations = { parent_silo: Silo, parent_fleet: Fleet };\n\n\t# Everyone who can read the Silo (which includes all the users in the\n\t# Silo) can see the identity providers in it.\n\t"list_children" if "read" on "parent_silo";\n\n\t# Fleet and Silo administrators can manage the Silo's identity provider\n\t# configuration. This is one of the only areas of Silo configuration\n\t# that Fleet Administrators have permissions on. This is also one of\n\t# the only cases where we need to look two levels up the hierarchy to\n\t# see if somebody has the right permission. For most other things,\n\t# permissions cascade down the hierarchy so we only need to look at the\n\t# parent.\n\t"create_child" if "admin" on "parent_silo";\n\t"create_child" if "admin" on "parent_fleet";\n}\nhas_relation(silo: Silo, "parent_silo", collection: SiloIdentityProviderList)\n\tif collection.silo = silo;\nhas_relation(fleet: Fleet, "parent_fleet", collection: SiloIdentityProviderList)\n\tif collection.silo.fleet = fleet;\n\n# Describes the policy for creating and managing Silo users (mostly intended for\n# API-managed users)\nresource SiloUserList {\n\tpermissions = [ "list_children", "create_child" ];\n\n\trelations = { parent_silo: Silo, parent_fleet: Fleet };\n\n\t# Everyone who can read the Silo (which includes all the users in the\n\t# Silo) can see the users in it.\n\t"list_children" if "read" on "parent_silo";\n\n\t# Fleet and Silo administrators can manage the Silo's users. This is\n\t# one of the only areas of Silo configuration that Fleet Administrators\n\t# have permissions on. This is also one of the few cases (so far) where\n\t# we need to look two levels up the hierarchy to see if somebody has the\n\t# right permission. For most other things, permissions cascade down the\n\t# hierarchy so we only need to look at the parent.\n\t"create_child" if "admin" on "parent_silo";\n\t"list_children" if "admin" on "parent_fleet";\n\t"create_child" if "admin" on "parent_fleet";\n}\nhas_relation(silo: Silo, "parent_silo", collection: SiloUserList)\n\tif collection.silo = silo;\nhas_relation(fleet: Fleet, "parent_fleet", collection: SiloUserList)\n\tif collection.silo.fleet = fleet;\n\n# These rules grants the external authenticator role the permissions it needs to\n# read silo users and modify their sessions. This is necessary for login to\n# work.\nhas_permission(actor: AuthenticatedActor, "read", silo: Silo)\n\tif has_role(actor, "external-authenticator", silo.fleet);\nhas_permission(actor: AuthenticatedActor, "read", user: SiloUser)\n\tif has_role(actor, "external-authenticator", user.silo.fleet);\nhas_permission(actor: AuthenticatedActor, "modify", user: SiloUser)\n\tif has_role(actor, "external-authenticator", user.silo.fleet);\nhas_permission(actor: AuthenticatedActor, "read", group: SiloGroup)\n\tif has_role(actor, "external-authenticator", group.silo.fleet);\nhas_permission(actor: AuthenticatedActor, "modify", group: SiloGroup)\n\tif has_role(actor, "external-authenticator", group.silo.fleet);\n\nhas_permission(actor: AuthenticatedActor, "read", session: ConsoleSession)\n\tif has_role(actor, "external-authenticator", session.fleet);\nhas_permission(actor: AuthenticatedActor, "modify", session: ConsoleSession)\n\tif has_role(actor, "external-authenticator", session.fleet);\n\n# All authenticated users can read and delete device authn requests because\n# by necessity these operations happen before we've figured out what user (or\n# even Silo) the device auth is associated with. Any user can claim a device\n# auth request with the right user code (that's how it works) -- it's the user\n# code and associated logic that prevents unauthorized access here.\nhas_permission(_actor: AuthenticatedActor, "read", _device_auth: DeviceAuthRequest);\nhas_permission(_actor: AuthenticatedActor, "modify", _device_auth: DeviceAuthRequest);\n\nhas_permission(actor: AuthenticatedActor, "read", device_token: DeviceAccessToken)\n\tif has_role(actor, "external-authenticator", device_token.fleet);\n\nhas_permission(actor: AuthenticatedActor, "read", identity_provider: IdentityProvider)\n\tif has_role(actor, "external-authenticator", identity_provider.silo.fleet);\n\nhas_permission(actor: AuthenticatedActor, "read", saml_identity_provider: SamlIdentityProvider)\n\tif has_role(actor, "external-authenticator", saml_identity_provider.silo.fleet);\n\n# Describes the policy for who can access the internal database.\nresource Database {\n\tpermissions = [\n\t # "query" is required to perform any query against the database,\n\t # whether a read or write query. This is checked when an operation\n\t # checks out a database connection from the connection pool.\n\t #\n\t # Any authenticated user gets this permission. There's generally\n\t # some other authz check involved in the database query. For\n\t # example, if you're querying the database to "read" a "Project", we\n\t # should also be checking that. So why do we do this at all? It's\n\t # a belt-and-suspenders measure so that if we somehow introduced an\n\t # unauthenticated code path that hits the database, it cannot be\n\t # used to DoS the database because we won't allow the operation to\n\t # make the query. (As long as the code path _is_ authenticated, we\n\t # can use throttling mechanisms to prevent DoS.)\n\t "query",\n\n\t # "modify" is required to populate database data that's delivered\n\t # with the system. It should also be required for schema changes,\n\t # when we support those. This is separate from "query" so that we\n\t # cannot accidentally invoke these code paths from API calls and\n\t # other general functions.\n\t "modify"\n\t];\n}\n\n# All authenticated users have the "query" permission on the database.\nhas_permission(_actor: AuthenticatedActor, "query", _resource: Database);\n\n# The "db-init" user is the only one with the "modify" permission.\nhas_permission(USER_DB_INIT: AuthenticatedActor, "modify", _resource: Database);\nhas_permission(USER_DB_INIT: AuthenticatedActor, "create_child", _resource: IpPoolList);\n# It also has "admin" on the internal silo to populate it with built-in resources.\n# TODO-completeness: actually limit to just internal silo and not all silos\nhas_role(USER_DB_INIT: AuthenticatedActor, "admin", _silo: Silo);\n\n# Allow the internal API admin permissions on all silos.\nhas_role(USER_INTERNAL_API: AuthenticatedActor, "admin", _silo: Silo);\n\n\n\n resource Disk {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n\n relations = { containing_project: Project };\n "list_children" if "viewer" on "containing_project";\n "read" if "viewer" on "containing_project";\n "modify" if "collaborator" on "containing_project";\n "create_child" if "collaborator" on "containing_project";\n }\n\n has_relation(parent: Project, "containing_project", child: Disk)\n if child.project = parent;\n \n\n resource Snapshot {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n\n relations = { containing_project: Project };\n "list_children" if "viewer" on "containing_project";\n "read" if "viewer" on "containing_project";\n "modify" if "collaborator" on "containing_project";\n "create_child" if "collaborator" on "containing_project";\n }\n\n has_relation(parent: Project, "containing_project", child: Snapshot)\n if child.project = parent;\n \n\n resource ProjectImage {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n\n relations = { containing_project: Project };\n "list_children" if "viewer" on "containing_project";\n "read" if "viewer" on "containing_project";\n "modify" if "collaborator" on "containing_project";\n "create_child" if "collaborator" on "containing_project";\n }\n\n has_relation(parent: Project, "containing_project", child: ProjectImage)\n if child.project = parent;\n \n\n resource Instance {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n\n relations = { containing_project: Project };\n "list_children" if "viewer" on "containing_project";\n "read" if "viewer" on "containing_project";\n "modify" if "collaborator" on "containing_project";\n "create_child" if "collaborator" on "containing_project";\n }\n\n has_relation(parent: Project, "containing_project", child: Instance)\n if child.project = parent;\n \n\n resource IpPool {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n \n relations = { parent_fleet: Fleet };\n "list_children" if "viewer" on "parent_fleet";\n "read" if "viewer" on "parent_fleet";\n "modify" if "admin" on "parent_fleet";\n "create_child" if "admin" on "parent_fleet";\n }\n has_relation(fleet: Fleet, "parent_fleet", child: IpPool)\n if child.fleet = fleet;\n \n\n resource InstanceNetworkInterface {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n\n relations = {\n containing_project: Project,\n parent: Instance\n };\n "list_children" if "viewer" on "containing_project";\n "read" if "viewer" on "containing_project";\n "modify" if "collaborator" on "containing_project";\n "create_child" if "collaborator" on "containing_project";\n }\n\n has_relation(project: Project, "containing_project", child: InstanceNetworkInterface)\n if has_relation(project, "containing_project", child.instance);\n\n has_relation(parent: Instance, "parent", child: InstanceNetworkInterface)\n if child.instance = parent;\n \n\n resource Vpc {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n\n relations = { containing_project: Project };\n "list_children" if "viewer" on "containing_project";\n "read" if "viewer" on "containing_project";\n "modify" if "collaborator" on "containing_project";\n "create_child" if "collaborator" on "containing_project";\n }\n\n has_relation(parent: Project, "containing_project", child: Vpc)\n if child.project = parent;\n \n\n resource VpcRouter {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n\n relations = {\n containing_project: Project,\n parent: Vpc\n };\n "list_children" if "viewer" on "containing_project";\n "read" if "viewer" on "containing_project";\n "modify" if "collaborator" on "containing_project";\n "create_child" if "collaborator" on "containing_project";\n }\n\n has_relation(project: Project, "containing_project", child: VpcRouter)\n if has_relation(project, "containing_project", child.vpc);\n\n has_relation(parent: Vpc, "parent", child: VpcRouter)\n if child.vpc = parent;\n \n\n resource InternetGateway {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n\n relations = {\n containing_project: Project,\n parent: Vpc\n };\n "list_children" if "viewer" on "containing_project";\n "read" if "viewer" on "containing_project";\n "modify" if "collaborator" on "containing_project";\n "create_child" if "collaborator" on "containing_project";\n }\n\n has_relation(project: Project, "containing_project", child: InternetGateway)\n if has_relation(project, "containing_project", child.vpc);\n\n has_relation(parent: Vpc, "parent", child: InternetGateway)\n if child.vpc = parent;\n \n\n resource InternetGatewayIpPool {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n\n relations = {\n containing_project: Project,\n parent: InternetGateway\n };\n "list_children" if "viewer" on "containing_project";\n "read" if "viewer" on "containing_project";\n "modify" if "collaborator" on "containing_project";\n "create_child" if "collaborator" on "containing_project";\n }\n\n has_relation(project: Project, "containing_project", child: InternetGatewayIpPool)\n if has_relation(project, "containing_project", child.internet_gateway);\n\n has_relation(parent: InternetGateway, "parent", child: InternetGatewayIpPool)\n if child.internet_gateway = parent;\n \n\n resource InternetGatewayIpAddress {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n\n relations = {\n containing_project: Project,\n parent: InternetGateway\n };\n "list_children" if "viewer" on "containing_project";\n "read" if "viewer" on "containing_project";\n "modify" if "collaborator" on "containing_project";\n "create_child" if "collaborator" on "containing_project";\n }\n\n has_relation(project: Project, "containing_project", child: InternetGatewayIpAddress)\n if has_relation(project, "containing_project", child.internet_gateway);\n\n has_relation(parent: InternetGateway, "parent", child: InternetGatewayIpAddress)\n if child.internet_gateway = parent;\n \n\n resource RouterRoute {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n\n relations = {\n containing_project: Project,\n parent: VpcRouter\n };\n "list_children" if "viewer" on "containing_project";\n "read" if "viewer" on "containing_project";\n "modify" if "collaborator" on "containing_project";\n "create_child" if "collaborator" on "containing_project";\n }\n\n has_relation(project: Project, "containing_project", child: RouterRoute)\n if has_relation(project, "containing_project", child.vpc_router);\n\n has_relation(parent: VpcRouter, "parent", child: RouterRoute)\n if child.vpc_router = parent;\n \n\n resource VpcSubnet {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n\n relations = {\n containing_project: Project,\n parent: Vpc\n };\n "list_children" if "viewer" on "containing_project";\n "read" if "viewer" on "containing_project";\n "modify" if "collaborator" on "containing_project";\n "create_child" if "collaborator" on "containing_project";\n }\n\n has_relation(project: Project, "containing_project", child: VpcSubnet)\n if has_relation(project, "containing_project", child.vpc);\n\n has_relation(parent: Vpc, "parent", child: VpcSubnet)\n if child.vpc = parent;\n \n\n resource FloatingIp {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n\n relations = { containing_project: Project };\n "list_children" if "viewer" on "containing_project";\n "read" if "viewer" on "containing_project";\n "modify" if "collaborator" on "containing_project";\n "create_child" if "collaborator" on "containing_project";\n }\n\n has_relation(parent: Project, "containing_project", child: FloatingIp)\n if child.project = parent;\n \n\n resource Image {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n\n relations = { containing_silo: Silo };\n "list_children" if "viewer" on "containing_silo";\n "read" if "viewer" on "containing_silo";\n "modify" if "collaborator" on "containing_silo";\n "create_child" if "collaborator" on "containing_silo";\n }\n\n has_relation(parent: Silo, "containing_silo", child: Image)\n if child.silo = parent;\n \n\n resource SiloImage {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n\n relations = { containing_silo: Silo };\n "list_children" if "viewer" on "containing_silo";\n "read" if "viewer" on "containing_silo";\n "modify" if "collaborator" on "containing_silo";\n "create_child" if "collaborator" on "containing_silo";\n }\n\n has_relation(parent: Silo, "containing_silo", child: SiloImage)\n if child.silo = parent;\n \n\n resource AddressLot {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n \n relations = { parent_fleet: Fleet };\n "list_children" if "viewer" on "parent_fleet";\n "read" if "viewer" on "parent_fleet";\n "modify" if "admin" on "parent_fleet";\n "create_child" if "admin" on "parent_fleet";\n }\n has_relation(fleet: Fleet, "parent_fleet", child: AddressLot)\n if child.fleet = fleet;\n \n\n resource Blueprint {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n \n relations = { parent_fleet: Fleet };\n "list_children" if "viewer" on "parent_fleet";\n "read" if "viewer" on "parent_fleet";\n "modify" if "admin" on "parent_fleet";\n "create_child" if "admin" on "parent_fleet";\n }\n has_relation(fleet: Fleet, "parent_fleet", child: Blueprint)\n if child.fleet = fleet;\n \n\n resource LoopbackAddress {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n \n relations = { parent_fleet: Fleet };\n "list_children" if "viewer" on "parent_fleet";\n "read" if "viewer" on "parent_fleet";\n "modify" if "admin" on "parent_fleet";\n "create_child" if "admin" on "parent_fleet";\n }\n has_relation(fleet: Fleet, "parent_fleet", child: LoopbackAddress)\n if child.fleet = fleet;\n \n\n\n resource ConsoleSession {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n \n relations = { parent_fleet: Fleet };\n "list_children" if "viewer" on "parent_fleet";\n "read" if "viewer" on "parent_fleet";\n "modify" if "admin" on "parent_fleet";\n "create_child" if "admin" on "parent_fleet";\n }\n has_relation(fleet: Fleet, "parent_fleet", child: ConsoleSession)\n if child.fleet = fleet;\n \n\n resource DeviceAuthRequest {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n \n relations = { parent_fleet: Fleet };\n "list_children" if "viewer" on "parent_fleet";\n "read" if "viewer" on "parent_fleet";\n "modify" if "admin" on "parent_fleet";\n "create_child" if "admin" on "parent_fleet";\n }\n has_relation(fleet: Fleet, "parent_fleet", child: DeviceAuthRequest)\n if child.fleet = fleet;\n \n\n resource DeviceAccessToken {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n \n relations = { parent_fleet: Fleet };\n "list_children" if "viewer" on "parent_fleet";\n "read" if "viewer" on "parent_fleet";\n "modify" if "admin" on "parent_fleet";\n "create_child" if "admin" on "parent_fleet";\n }\n has_relation(fleet: Fleet, "parent_fleet", child: DeviceAccessToken)\n if child.fleet = fleet;\n \n\n resource PhysicalDisk {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n \n relations = { parent_fleet: Fleet };\n "list_children" if "viewer" on "parent_fleet";\n "read" if "viewer" on "parent_fleet";\n "modify" if "admin" on "parent_fleet";\n "create_child" if "admin" on "parent_fleet";\n }\n has_relation(fleet: Fleet, "parent_fleet", child: PhysicalDisk)\n if child.fleet = fleet;\n \n\n resource Rack {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n \n relations = { parent_fleet: Fleet };\n "list_children" if "viewer" on "parent_fleet";\n "read" if "viewer" on "parent_fleet";\n "modify" if "admin" on "parent_fleet";\n "create_child" if "admin" on "parent_fleet";\n }\n has_relation(fleet: Fleet, "parent_fleet", child: Rack)\n if child.fleet = fleet;\n \n\n resource RoleBuiltin {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n \n relations = { parent_fleet: Fleet };\n "list_children" if "viewer" on "parent_fleet";\n "read" if "viewer" on "parent_fleet";\n "modify" if "admin" on "parent_fleet";\n "create_child" if "admin" on "parent_fleet";\n }\n has_relation(fleet: Fleet, "parent_fleet", child: RoleBuiltin)\n if child.fleet = fleet;\n \n\n\n\n\n\n\n\n resource Sled {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n \n relations = { parent_fleet: Fleet };\n "list_children" if "viewer" on "parent_fleet";\n "read" if "viewer" on "parent_fleet";\n "modify" if "admin" on "parent_fleet";\n "create_child" if "admin" on "parent_fleet";\n }\n has_relation(fleet: Fleet, "parent_fleet", child: Sled)\n if child.fleet = fleet;\n \n\n resource TufRepo {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n \n relations = { parent_fleet: Fleet };\n "list_children" if "viewer" on "parent_fleet";\n "read" if "viewer" on "parent_fleet";\n "modify" if "admin" on "parent_fleet";\n "create_child" if "admin" on "parent_fleet";\n }\n has_relation(fleet: Fleet, "parent_fleet", child: TufRepo)\n if child.fleet = fleet;\n \n\n resource TufArtifact {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n \n relations = { parent_fleet: Fleet };\n "list_children" if "viewer" on "parent_fleet";\n "read" if "viewer" on "parent_fleet";\n "modify" if "admin" on "parent_fleet";\n "create_child" if "admin" on "parent_fleet";\n }\n has_relation(fleet: Fleet, "parent_fleet", child: TufArtifact)\n if child.fleet = fleet;\n \n\n resource Zpool {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n \n relations = { parent_fleet: Fleet };\n "list_children" if "viewer" on "parent_fleet";\n "read" if "viewer" on "parent_fleet";\n "modify" if "admin" on "parent_fleet";\n "create_child" if "admin" on "parent_fleet";\n }\n has_relation(fleet: Fleet, "parent_fleet", child: Zpool)\n if child.fleet = fleet;\n \n\n resource Service {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n \n relations = { parent_fleet: Fleet };\n "list_children" if "viewer" on "parent_fleet";\n "read" if "viewer" on "parent_fleet";\n "modify" if "admin" on "parent_fleet";\n "create_child" if "admin" on "parent_fleet";\n }\n has_relation(fleet: Fleet, "parent_fleet", child: Service)\n if child.fleet = fleet;\n \n\n resource UserBuiltin {\n permissions = [\n "list_children",\n "modify",\n "read",\n "create_child",\n ];\n \n relations = { parent_fleet: Fleet };\n "list_children" if "viewer" on "parent_fleet";\n "read" if "viewer" on "parent_fleet";\n "modify" if "admin" on "parent_fleet";\n "create_child" if "admin" on "parent_fleet";\n }\n has_relation(fleet: Fleet, "parent_fleet", child: UserBuiltin)\n if child.fleet = fleet;\n file = nexus/auth/src/authz/oso_generic.rs:89
632024-11-21T19:33:13.371ZINFOnexus (ServerContext): Setting up resolver using DNS servers for subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344::, width: 48 } } file = nexus/src/context.rs:219
642024-11-21T19:33:13.374ZINFOnexus (DnsResolver): new DNS resolver addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53] file = internal-dns/resolver/src/resolver.rs:111
652024-11-21T19:33:13.378ZINFOnexus (ServerContext): Setting up qorb database pool from DNS dns_addrs = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53] file = nexus/src/context.rs:275
662024-11-21T19:33:13.384ZDEBGnexus (ServerContext): registered USDT probes
672024-11-21T19:33:13.618ZINFOnexus: Database schema version is up to date desired_version = 114.0.0 file = nexus/db-queries/src/db/datastore/db_metadata.rs:145 found_version = 114.0.0
682024-11-21T19:33:13.620ZINFOnexus (SEC): SEC running file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/steno-0.4.1/src/sec.rs:813 sec_id = 915e11b8-d7cb-4730-9906-7980540813dc
692024-11-21T19:33:13.620ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 736, tv_nsec: 501821624 } })
702024-11-21T19:33:13.621ZINFOnexus (Nexus): Determining switch slots managed by switch zones file = nexus/src/app/mod.rs:1131
712024-11-21T19:33:13.670ZINFOnexus (Nexus): determining switch slot managed by dendrite zone file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
722024-11-21T19:33:13.670ZDEBGnexus (MgsClient): client request body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
732024-11-21T19:33:13.674ZDEBGnexus (MgsClient): client response result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "67098fab-2460-4579-aa26-ddab557f88b2", "content-length": "26", "date": "Thu, 21 Nov 2024 19:33:13 GMT"} })
742024-11-21T19:33:13.674ZINFOnexus (Nexus): identified switch slot for dendrite zone file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
752024-11-21T19:33:13.675ZINFOnexus (Nexus): completed mapping dendrite zones to switch slots file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
762024-11-21T19:33:13.706ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 736, tv_nsec: 587553730 } })
772024-11-21T19:33:13.706ZINFOnexus (Nexus): Determining switch slots managed by switch zones file = nexus/src/app/mod.rs:1131
782024-11-21T19:33:13.739ZINFOnexus (Nexus): determining switch slot managed by dendrite zone file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
792024-11-21T19:33:13.739ZDEBGnexus (MgsClient): client request body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
802024-11-21T19:33:13.740ZDEBGnexus (MgsClient): client response result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "69be3073-8f7e-4b26-8896-e2bfad2a1462", "content-length": "26", "date": "Thu, 21 Nov 2024 19:33:13 GMT"} })
812024-11-21T19:33:13.740ZINFOnexus (Nexus): identified switch slot for dendrite zone file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
822024-11-21T19:33:13.741ZINFOnexus (Nexus): completed mapping dendrite zones to switch slots file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
832024-11-21T19:33:13.808ZDEBGnexus (clickhouse-client): registered USDT probes id = dc2971fe-8e9f-438a-981d-130fe715bfbe
842024-11-21T19:33:13.809ZDEBGnexus (clickhouse-client): registered USDT probes id = dc2971fe-8e9f-438a-981d-130fe715bfbe
852024-11-21T19:33:13.824ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
862024-11-21T19:33:13.837ZDEBGnexus (DataLoader): authorize result action = Modify actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
872024-11-21T19:33:13.838ZDEBGnexus (DataLoader): attempting to create built-in users
882024-11-21T19:33:13.838ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
892024-11-21T19:33:13.839ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
902024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = GET path = /artifacts/{kind}/{name}/{version} versions = all
912024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = GET path = /bgtasks versions = all
922024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /bgtasks/activate versions = all
932024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = GET path = /bgtasks/view/{bgtask_name} versions = all
942024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = GET path = /clickhouse/policy versions = all
952024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /clickhouse/policy versions = all
962024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /crucible/0/upstairs/{upstairs_id}/downstairs/{downstairs_id}/stop-request versions = all
972024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /crucible/0/upstairs/{upstairs_id}/downstairs/{downstairs_id}/stopped versions = all
982024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /crucible/0/upstairs/{upstairs_id}/repair/{repair_id}/progress versions = all
992024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /crucible/0/upstairs/{upstairs_id}/repair-finish versions = all
1002024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /crucible/0/upstairs/{upstairs_id}/repair-start versions = all
1012024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /demo-saga versions = all
1022024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /demo-saga/{demo_saga_id}/complete versions = all
1032024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = GET path = /deployment/blueprints/all versions = all
1042024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = DELETE path = /deployment/blueprints/all/{blueprint_id} versions = all
1052024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = GET path = /deployment/blueprints/all/{blueprint_id} versions = all
1062024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /deployment/blueprints/import versions = all
1072024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /deployment/blueprints/regenerate versions = all
1082024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = GET path = /deployment/blueprints/target versions = all
1092024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /deployment/blueprints/target versions = all
1102024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = PUT path = /deployment/blueprints/target/enabled versions = all
1112024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /disk/{disk_id}/remove-read-only-parent versions = all
1122024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = PUT path = /disks/{disk_id} versions = all
1132024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /instances/{instance_id}/migrate versions = all
1142024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /metrics/collectors versions = all
1152024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = GET path = /metrics/collectors/{collector_id}/producers versions = all
1162024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /metrics/producers versions = all
1172024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = GET path = /nat/ipv4/changeset/{from_gen} versions = all
1182024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /physical-disk/expunge versions = all
1192024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = GET path = /probes/{sled} versions = all
1202024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = PUT path = /racks/{rack_id}/initialization-complete versions = all
1212024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = GET path = /sagas versions = all
1222024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = GET path = /sagas/{saga_id} versions = all
1232024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = GET path = /sled-agents/{sled_id} versions = all
1242024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /sled-agents/{sled_id} versions = all
1252024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /sled-agents/{sled_id}/firewall-rules-update versions = all
1262024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /sleds/add versions = all
1272024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /sleds/expunge versions = all
1282024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = GET path = /sleds/uninitialized versions = all
1292024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = PUT path = /switch/{switch_id} versions = all
1302024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = PUT path = /vmms/{propolis_id} versions = all
1312024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): registered endpoint local_addr = [fd00:1122:3344:101::c]:12221 method = POST path = /volume/{volume_id}/remove-read-only-parent versions = all
1322024-11-21T19:33:13.855ZINFOnexus (dropshot_internal): listening file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327 local_addr = [fd00:1122:3344:101::c]:12221
1332024-11-21T19:33:13.855ZDEBGnexus (dropshot_internal): successfully registered DTrace USDT probes local_addr = [fd00:1122:3344:101::c]:12221
1342024-11-21T19:33:13.856ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
1352024-11-21T19:33:13.856ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
1362024-11-21T19:33:13.891ZWARNnexus: Cannot look up rack: Object (of type ById(6f50eaaf-6509-4592-a570-21594f7b8789)) not found: rack file = nexus/src/app/rack.rs:774
1372024-11-21T19:33:14.247ZINFOnexus (DataLoader): created 5 built-in users file = nexus/db-queries/src/db/datastore/silo_user.rs:394
1382024-11-21T19:33:14.247ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1392024-11-21T19:33:14.249ZDEBGnexus (DataLoader): authorize result action = Modify actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1402024-11-21T19:33:14.249ZDEBGnexus (DataLoader): attempting to create built-in roles
1412024-11-21T19:33:14.249ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1422024-11-21T19:33:14.249ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1432024-11-21T19:33:14.673ZINFOnexus (DataLoader): created 10 built-in roles file = nexus/db-queries/src/db/datastore/role.rs:85
1442024-11-21T19:33:14.673ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1452024-11-21T19:33:14.675ZDEBGnexus (DataLoader): authorize result action = Modify actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1462024-11-21T19:33:14.675ZDEBGnexus (DataLoader): attempting to create built-in role assignments
1472024-11-21T19:33:14.675ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1482024-11-21T19:33:14.675ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1492024-11-21T19:33:15.440ZINFOnexus (DataLoader): created 4 built-in role assignments file = nexus/db-queries/src/db/datastore/role.rs:114
1502024-11-21T19:33:15.440ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1512024-11-21T19:33:15.442ZDEBGnexus (DataLoader): authorize result action = Modify actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1522024-11-21T19:33:15.442ZDEBGnexus (DataLoader): attempting to create built-in silos
1532024-11-21T19:33:15.442ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1542024-11-21T19:33:15.442ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1552024-11-21T19:33:15.741ZINFOnexus (DataLoader): created 2 built-in silos file = nexus/db-queries/src/db/datastore/silo.rs:95
1562024-11-21T19:33:15.741ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1572024-11-21T19:33:15.741ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1582024-11-21T19:33:15.765ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1592024-11-21T19:33:15.767ZDEBGnexus (DataLoader): authorize result action = Modify actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1602024-11-21T19:33:15.767ZDEBGnexus (DataLoader): attempting to create built-in projects
1612024-11-21T19:33:15.767ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1622024-11-21T19:33:15.767ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1632024-11-21T19:33:15.799ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1642024-11-21T19:33:15.800ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1652024-11-21T19:33:15.893ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
1662024-11-21T19:33:15.893ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
1672024-11-21T19:33:15.895ZWARNnexus: Cannot look up rack: Object (of type ById(6f50eaaf-6509-4592-a570-21594f7b8789)) not found: rack file = nexus/src/app/rack.rs:774
1682024-11-21T19:33:15.974ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1692024-11-21T19:33:15.974ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1702024-11-21T19:33:16.009ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1712024-11-21T19:33:16.011ZDEBGnexus (DataLoader): authorize result action = CreateChild actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) } result = Ok(())
1722024-11-21T19:33:16.011ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1732024-11-21T19:33:16.011ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1742024-11-21T19:33:16.048ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1752024-11-21T19:33:16.049ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1762024-11-21T19:33:16.051ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1772024-11-21T19:33:16.053ZDEBGnexus (DataLoader): authorize result action = CreateChild actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) } result = Ok(())
1782024-11-21T19:33:16.053ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1792024-11-21T19:33:16.053ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1802024-11-21T19:33:16.170ZINFOnexus (DataLoader): created built-in services project file = nexus/db-queries/src/db/datastore/project.rs:122
1812024-11-21T19:33:16.170ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1822024-11-21T19:33:16.171ZDEBGnexus (DataLoader): authorize result action = Modify actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1832024-11-21T19:33:16.171ZDEBGnexus (DataLoader): attempting to create built-in VPCs
1842024-11-21T19:33:16.171ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1852024-11-21T19:33:16.171ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1862024-11-21T19:33:16.215ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1872024-11-21T19:33:16.216ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1882024-11-21T19:33:16.219ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1892024-11-21T19:33:16.219ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1902024-11-21T19:33:16.224ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1912024-11-21T19:33:16.224ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1922024-11-21T19:33:16.228ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1932024-11-21T19:33:16.228ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1942024-11-21T19:33:16.230ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1952024-11-21T19:33:16.232ZDEBGnexus (DataLoader): authorize result action = CreateChild actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) } result = Ok(())
1962024-11-21T19:33:16.232ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1972024-11-21T19:33:16.232ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
1982024-11-21T19:33:16.552ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
1992024-11-21T19:33:16.552ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2002024-11-21T19:33:16.554ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2012024-11-21T19:33:16.554ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2022024-11-21T19:33:16.556ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2032024-11-21T19:33:16.558ZDEBGnexus (DataLoader): authorize result action = CreateChild actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) } result = Ok(())
2042024-11-21T19:33:16.558ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2052024-11-21T19:33:16.558ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2062024-11-21T19:33:16.671ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2072024-11-21T19:33:16.671ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2082024-11-21T19:33:16.673ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2092024-11-21T19:33:16.673ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2102024-11-21T19:33:16.675ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2112024-11-21T19:33:16.675ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2122024-11-21T19:33:16.676ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2132024-11-21T19:33:16.679ZDEBGnexus (DataLoader): authorize result action = CreateChild actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") } result = Ok(())
2142024-11-21T19:33:16.679ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2152024-11-21T19:33:16.679ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2162024-11-21T19:33:16.695ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2172024-11-21T19:33:16.696ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2182024-11-21T19:33:16.708ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2192024-11-21T19:33:16.708ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2202024-11-21T19:33:16.718ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2212024-11-21T19:33:16.718ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2222024-11-21T19:33:16.720ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2232024-11-21T19:33:16.720ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2242024-11-21T19:33:16.722ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2252024-11-21T19:33:16.722ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2262024-11-21T19:33:16.723ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2272024-11-21T19:33:16.726ZDEBGnexus (DataLoader): authorize result action = CreateChild actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") } result = Ok(())
2282024-11-21T19:33:16.726ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2292024-11-21T19:33:16.726ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2302024-11-21T19:33:16.766ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2312024-11-21T19:33:16.767ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2322024-11-21T19:33:16.768ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2332024-11-21T19:33:16.768ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2342024-11-21T19:33:16.770ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2352024-11-21T19:33:16.770ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2362024-11-21T19:33:16.772ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2372024-11-21T19:33:16.777ZDEBGnexus (DataLoader): authorize result action = CreateChild actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = VpcRouter { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-074c-4000-8000-000000000001, lookup_type: ById(001de000-074c-4000-8000-000000000001) } result = Ok(())
2382024-11-21T19:33:16.777ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2392024-11-21T19:33:16.777ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2402024-11-21T19:33:16.851ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2412024-11-21T19:33:16.851ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2422024-11-21T19:33:16.853ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2432024-11-21T19:33:16.853ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2442024-11-21T19:33:16.854ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2452024-11-21T19:33:16.855ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2462024-11-21T19:33:16.856ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2472024-11-21T19:33:16.859ZDEBGnexus (DataLoader): authorize result action = CreateChild actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = VpcRouter { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-074c-4000-8000-000000000001, lookup_type: ById(001de000-074c-4000-8000-000000000001) } result = Ok(())
2482024-11-21T19:33:16.859ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2492024-11-21T19:33:16.859ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2502024-11-21T19:33:16.957ZDEBGnexus (DataLoader): attempting to create built-in VPC firewall rules
2512024-11-21T19:33:16.957ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2522024-11-21T19:33:16.958ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2532024-11-21T19:33:16.976ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2542024-11-21T19:33:16.977ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2552024-11-21T19:33:16.978ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2562024-11-21T19:33:16.979ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2572024-11-21T19:33:16.981ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2582024-11-21T19:33:16.981ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2592024-11-21T19:33:16.987ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2602024-11-21T19:33:16.987ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2612024-11-21T19:33:16.989ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2622024-11-21T19:33:16.989ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2632024-11-21T19:33:16.990ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2642024-11-21T19:33:16.993ZDEBGnexus (DataLoader): authorize result action = CreateChild actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
2652024-11-21T19:33:16.993ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2662024-11-21T19:33:16.993ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2672024-11-21T19:33:16.995ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2682024-11-21T19:33:16.995ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2692024-11-21T19:33:16.996ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2702024-11-21T19:33:16.996ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2712024-11-21T19:33:17.003ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2722024-11-21T19:33:17.006ZDEBGnexus (DataLoader): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
2732024-11-21T19:33:17.006ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2742024-11-21T19:33:17.006ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2752024-11-21T19:33:17.031ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2762024-11-21T19:33:17.031ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2772024-11-21T19:33:17.032ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2782024-11-21T19:33:17.033ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2792024-11-21T19:33:17.039ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2802024-11-21T19:33:17.040ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2812024-11-21T19:33:17.046ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2822024-11-21T19:33:17.048ZDEBGnexus (DataLoader): authorize result action = Modify actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
2832024-11-21T19:33:17.049ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2842024-11-21T19:33:17.049ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2852024-11-21T19:33:17.164ZINFOnexus (DataLoader): created built-in VPC firewall rules file = nexus/db-queries/src/db/datastore/vpc.rs:282
2862024-11-21T19:33:17.164ZDEBGnexus (DataLoader): attempting to create built-in VPC Subnets
2872024-11-21T19:33:17.164ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2882024-11-21T19:33:17.164ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2892024-11-21T19:33:17.176ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2902024-11-21T19:33:17.177ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2912024-11-21T19:33:17.179ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2922024-11-21T19:33:17.179ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2932024-11-21T19:33:17.181ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2942024-11-21T19:33:17.181ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2952024-11-21T19:33:17.183ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2962024-11-21T19:33:17.183ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2972024-11-21T19:33:17.184ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
2982024-11-21T19:33:17.185ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
2992024-11-21T19:33:17.186ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3002024-11-21T19:33:17.189ZDEBGnexus (DataLoader): authorize result action = CreateChild actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
3012024-11-21T19:33:17.189ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3022024-11-21T19:33:17.189ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3032024-11-21T19:33:17.205ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3042024-11-21T19:33:17.205ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3052024-11-21T19:33:17.207ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3062024-11-21T19:33:17.207ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3072024-11-21T19:33:17.208ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3082024-11-21T19:33:17.208ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3092024-11-21T19:33:17.210ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3102024-11-21T19:33:17.213ZDEBGnexus (DataLoader): authorize result action = CreateChild actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
3112024-11-21T19:33:17.241ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3122024-11-21T19:33:17.242ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3132024-11-21T19:33:17.279ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3142024-11-21T19:33:17.280ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3152024-11-21T19:33:17.294ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3162024-11-21T19:33:17.294ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3172024-11-21T19:33:17.295ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3182024-11-21T19:33:17.295ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3192024-11-21T19:33:17.297ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3202024-11-21T19:33:17.297ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3212024-11-21T19:33:17.299ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3222024-11-21T19:33:17.303ZDEBGnexus (DataLoader): authorize result action = CreateChild actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = VpcRouter { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-074c-4000-8000-000000000001, lookup_type: ById(001de000-074c-4000-8000-000000000001) } result = Ok(())
3232024-11-21T19:33:17.303ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3242024-11-21T19:33:17.303ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3252024-11-21T19:33:17.331ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3262024-11-21T19:33:17.332ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3272024-11-21T19:33:17.341ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3282024-11-21T19:33:17.341ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3292024-11-21T19:33:17.342ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3302024-11-21T19:33:17.343ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3312024-11-21T19:33:17.344ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3322024-11-21T19:33:17.344ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3332024-11-21T19:33:17.346ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3342024-11-21T19:33:17.348ZDEBGnexus (DataLoader): authorize result action = CreateChild actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
3352024-11-21T19:33:17.378ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3362024-11-21T19:33:17.379ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3372024-11-21T19:33:17.429ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3382024-11-21T19:33:17.429ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3392024-11-21T19:33:17.435ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3402024-11-21T19:33:17.436ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3412024-11-21T19:33:17.437ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3422024-11-21T19:33:17.437ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3432024-11-21T19:33:17.439ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3442024-11-21T19:33:17.439ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3452024-11-21T19:33:17.440ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3462024-11-21T19:33:17.444ZDEBGnexus (DataLoader): authorize result action = CreateChild actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = VpcRouter { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-074c-4000-8000-000000000001, lookup_type: ById(001de000-074c-4000-8000-000000000001) } result = Ok(())
3472024-11-21T19:33:17.444ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3482024-11-21T19:33:17.444ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3492024-11-21T19:33:17.454ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3502024-11-21T19:33:17.454ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3512024-11-21T19:33:17.456ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3522024-11-21T19:33:17.456ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3532024-11-21T19:33:17.457ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3542024-11-21T19:33:17.458ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3552024-11-21T19:33:17.459ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3562024-11-21T19:33:17.460ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3572024-11-21T19:33:17.461ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3582024-11-21T19:33:17.462ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3592024-11-21T19:33:17.465ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3602024-11-21T19:33:17.465ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3612024-11-21T19:33:17.466ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3622024-11-21T19:33:17.466ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3632024-11-21T19:33:17.470ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3642024-11-21T19:33:17.474ZDEBGnexus (DataLoader): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) }, key: 001de000-c470-4000-8000-000000000003, lookup_type: ById(001de000-c470-4000-8000-000000000003) } result = Ok(())
3652024-11-21T19:33:17.474ZINFOnexus (DataLoader): created built-in services vpc subnets file = nexus/db-queries/src/db/datastore/vpc.rs:346
3662024-11-21T19:33:17.474ZINFOnexus (DataLoader): created built-in services vpc file = nexus/db-queries/src/db/datastore/vpc.rs:224
3672024-11-21T19:33:17.474ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3682024-11-21T19:33:17.475ZDEBGnexus (DataLoader): authorize result action = Modify actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3692024-11-21T19:33:17.475ZDEBGnexus (DataLoader): attempting to create silo users
3702024-11-21T19:33:17.475ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3712024-11-21T19:33:17.475ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3722024-11-21T19:33:17.490ZINFOnexus (DataLoader): created 2 silo users file = nexus/db-queries/src/db/datastore/silo_user.rs:419
3732024-11-21T19:33:17.490ZDEBGnexus (DataLoader): attempting to create silo user role assignments
3742024-11-21T19:33:17.490ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3752024-11-21T19:33:17.490ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3762024-11-21T19:33:17.500ZINFOnexus (DataLoader): created 2 silo user role assignments file = nexus/db-queries/src/db/datastore/silo_user.rs:446
3772024-11-21T19:33:17.500ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3782024-11-21T19:33:17.500ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3792024-11-21T19:33:17.521ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3802024-11-21T19:33:17.521ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3812024-11-21T19:33:17.528ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3822024-11-21T19:33:17.529ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3832024-11-21T19:33:17.530ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3842024-11-21T19:33:17.531ZDEBGnexus (DataLoader): authorize result action = CreateChild actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = IpPoolList result = Ok(())
3852024-11-21T19:33:17.531ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3862024-11-21T19:33:17.532ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3872024-11-21T19:33:17.549ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3882024-11-21T19:33:17.550ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3892024-11-21T19:33:17.551ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3902024-11-21T19:33:17.552ZDEBGnexus (DataLoader): authorize result action = CreateChild actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = IpPoolList result = Ok(())
3912024-11-21T19:33:17.552ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3922024-11-21T19:33:17.552ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3932024-11-21T19:33:17.809ZDEBGnexus (DataLoader): roles roles = RoleSet { roles: {} }
3942024-11-21T19:33:17.809ZDEBGnexus (DataLoader): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000001, .. }) resource = Database result = Ok(())
3952024-11-21T19:33:17.820ZINFOnexus: populate complete file = nexus/src/app/mod.rs:555
3962024-11-21T19:33:17.820ZINFOnexus: activating background tasks file = nexus/src/app/mod.rs:564
3972024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 1 reason = Timeout
3982024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 1 reason = Timeout
3992024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 1 reason = Timeout
4002024-11-21T19:33:17.829ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:74 reason = no blueprint
4012024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 1 reason = Timeout
4022024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = saga_recovery iteration = 1 reason = Timeout
4032024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 1 reason = Timeout
4042024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = instance_reincarnation iteration = 1 reason = Timeout
4052024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
4062024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 1 reason = Timeout
4072024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 1 reason = Timeout
4082024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 1 reason = Timeout
4092024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 1 reason = Timeout
4102024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 1 reason = Timeout
4112024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 1 reason = Timeout
4122024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 1 reason = Timeout
4132024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 1 reason = Timeout
4142024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 1 reason = Timeout
4152024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 1 reason = Timeout
4162024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 1 reason = Timeout
4172024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 1 reason = Timeout
4182024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 1 reason = Timeout
4192024-11-21T19:33:17.829ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
4202024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
4212024-11-21T19:33:17.829ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-11-21 19:23:17.829084625 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
4222024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
4232024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 1 reason = Timeout
4242024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 1 reason = Timeout
4252024-11-21T19:33:17.829ZWARNnexus (BackgroundTasks): DNS propagation: skipped background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:61 reason = no config nor servers
4262024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 1.271117ms iteration = 1 status = {"error":"no config nor servers"}
4272024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 1.207016ms iteration = 1 status = {"error":"no blueprint"}
4282024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
4292024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
4302024-11-21T19:33:17.829ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
4312024-11-21T19:33:17.830ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
4322024-11-21T19:33:17.830ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
4332024-11-21T19:33:17.830ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
4342024-11-21T19:33:17.830ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 1 reason = Timeout
4352024-11-21T19:33:17.830ZDEBGnexus (BackgroundTasks): activating background_task = service_firewall_rule_propagation iteration = 1 reason = Timeout
4362024-11-21T19:33:17.830ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
4372024-11-21T19:33:17.830ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
4382024-11-21T19:33:17.830ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
4392024-11-21T19:33:17.830ZDEBGnexus (service-firewall-rule-progator): starting background task for service firewall rule propagation background_task = service_firewall_rule_propagation
4402024-11-21T19:33:17.830ZWARNnexus (BackgroundTasks): DNS propagation: skipped background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:61 reason = no config nor servers
4412024-11-21T19:33:17.830ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 2.762557ms iteration = 1 status = {"error":"no config nor servers"}
4422024-11-21T19:33:17.831ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
4432024-11-21T19:33:17.831ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
4442024-11-21T19:33:17.831ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 1 reason = Timeout
4452024-11-21T19:33:17.831ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
4462024-11-21T19:33:17.831ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
4472024-11-21T19:33:17.831ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
4482024-11-21T19:33:17.831ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
4492024-11-21T19:33:17.831ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
4502024-11-21T19:33:17.831ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
4512024-11-21T19:33:17.831ZDEBGnexus (SagaRecoverer): listing candidate sagas for recovery
4522024-11-21T19:33:17.831ZDEBGnexus (SagaRecoverer): roles roles = RoleSet { roles: {} }
4532024-11-21T19:33:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
4542024-11-21T19:33:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
4552024-11-21T19:33:17.831ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
4562024-11-21T19:33:17.831ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 1 reason = Timeout
4572024-11-21T19:33:17.831ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
4582024-11-21T19:33:17.831ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 1 reason = Timeout
4592024-11-21T19:33:17.831ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 1 reason = Timeout
4602024-11-21T19:33:17.832ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
4612024-11-21T19:33:17.832ZDEBGnexus (BackgroundTasks): activating background_task = inventory_collection iteration = 1 reason = Timeout
4622024-11-21T19:33:17.832ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 1 reason = Timeout
4632024-11-21T19:33:17.832ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
4642024-11-21T19:33:17.832ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {} }
4652024-11-21T19:33:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
4662024-11-21T19:33:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
4672024-11-21T19:33:17.834ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
4682024-11-21T19:33:17.834ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
4692024-11-21T19:33:17.834ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
4702024-11-21T19:33:17.834ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 1 reason = Timeout
4712024-11-21T19:33:17.834ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 1 reason = Timeout
4722024-11-21T19:33:17.834ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 1 reason = Timeout
4732024-11-21T19:33:17.834ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
4742024-11-21T19:33:17.834ZDEBGnexus (BackgroundTasks): activating background_task = crdb_node_id_collector iteration = 1 reason = Timeout
4752024-11-21T19:33:17.834ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = crdb_node_id_collector file = nexus/src/app/background/tasks/crdb_node_id_collector.rs:78 reason = no blueprint
4762024-11-21T19:33:17.834ZDEBGnexus (BackgroundTasks): activation complete background_task = crdb_node_id_collector elapsed = 6.518987ms iteration = 1 status = {"error":"no blueprint"}
4772024-11-21T19:33:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
4782024-11-21T19:33:17.834ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
4792024-11-21T19:33:17.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
4802024-11-21T19:33:17.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
4812024-11-21T19:33:17.835ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-11-21 19:23:17.829084625 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
4822024-11-21T19:33:17.835ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 7.033325ms iteration = 1 status = {"expiration":"2024-11-21T19:23:17.829084625Z","pruned":[]}
4832024-11-21T19:33:17.835ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
4842024-11-21T19:33:17.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
4852024-11-21T19:33:17.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
4862024-11-21T19:33:17.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
4872024-11-21T19:33:17.836ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("4e669ff4-6dde-4cd2-8491-623460b56d8d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 740, tv_nsec: 717271397 } })
4882024-11-21T19:33:17.836ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0377cf96-baf3-4e09-a570-eb7c67037d4d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0d1b3b31-4591-4bcb-931d-e8e03365f0d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:1::1))) }], valid_until: Instant { tv_sec: 740, tv_nsec: 717478150 } })
4892024-11-21T19:33:17.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
4902024-11-21T19:33:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
4912024-11-21T19:33:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
4922024-11-21T19:33:17.837ZINFOnexus (BackgroundTasks): found DNS servers (initial) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external dns_group = external file = nexus/src/app/background/tasks/dns_servers.rs:113
4932024-11-21T19:33:17.837ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 8.95201ms iteration = 1 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
4942024-11-21T19:33:17.837ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 2 reason = Dependency
4952024-11-21T19:33:17.837ZWARNnexus (BackgroundTasks): DNS propagation: skipped background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:69 reason = no config
4962024-11-21T19:33:17.837ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 4.95\u{b5}s iteration = 2 status = {"error":"no config"}
4972024-11-21T19:33:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
4982024-11-21T19:33:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
4992024-11-21T19:33:17.837ZINFOnexus (BackgroundTasks): found DNS servers (initial) addresses = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } background_task = dns_servers_internal dns_group = internal file = nexus/src/app/background/tasks/dns_servers.rs:113
5002024-11-21T19:33:17.837ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 9.317924ms iteration = 1 status = {"addresses":["[fd00:1122:3344:3::1]:5353","[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353"]}
5012024-11-21T19:33:17.837ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 2 reason = Dependency
5022024-11-21T19:33:17.837ZWARNnexus (BackgroundTasks): DNS propagation: skipped background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:69 reason = no config
5032024-11-21T19:33:17.837ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 3.61\u{b5}s iteration = 2 status = {"error":"no config"}
5042024-11-21T19:33:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
5052024-11-21T19:33:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
5062024-11-21T19:33:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
5072024-11-21T19:33:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
5082024-11-21T19:33:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
5092024-11-21T19:33:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Database result = Ok(())
5102024-11-21T19:33:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
5112024-11-21T19:33:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
5122024-11-21T19:33:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
5132024-11-21T19:33:17.838ZDEBGnexus (SagaRecoverer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000003a8a, .. }) resource = Database result = Ok(())
5142024-11-21T19:33:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
5152024-11-21T19:33:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
5162024-11-21T19:33:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
5172024-11-21T19:33:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
5182024-11-21T19:33:17.839ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
5192024-11-21T19:33:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
5202024-11-21T19:33:17.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
5212024-11-21T19:33:17.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
5222024-11-21T19:33:17.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
5232024-11-21T19:33:17.841ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
5242024-11-21T19:33:17.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
5252024-11-21T19:33:17.846ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
5262024-11-21T19:33:17.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
5272024-11-21T19:33:17.848ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
5282024-11-21T19:33:17.848ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
5292024-11-21T19:33:17.854ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 25.668552ms iteration = 1 status = {"errors":[],"found_port_ok":[]}
5302024-11-21T19:33:17.870ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 740, tv_nsec: 751629516 } })
5312024-11-21T19:33:17.872ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
5322024-11-21T19:33:17.896ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
5332024-11-21T19:33:17.897ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
5342024-11-21T19:33:17.901ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5352024-11-21T19:33:17.903ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 75.200945ms iteration = 1 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
5362024-11-21T19:33:17.904ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5372024-11-21T19:33:17.904ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
5382024-11-21T19:33:17.904ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
5392024-11-21T19:33:17.904ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
5402024-11-21T19:33:17.904ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
5412024-11-21T19:33:17.905ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
5422024-11-21T19:33:17.908ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
5432024-11-21T19:33:17.908ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
5442024-11-21T19:33:17.908ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
5452024-11-21T19:33:17.908ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
5462024-11-21T19:33:17.909ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
5472024-11-21T19:33:17.909ZWARNnexus (BackgroundTasks): failed to read target blueprint background_task = blueprint_loader error = Internal Error: no target blueprint set file = nexus/src/app/background/tasks/blueprint_load.rs:69
5482024-11-21T19:33:17.909ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 81.198194ms iteration = 1 status = {"error":"failed to read target blueprint: Internal Error: no target blueprint set"}
5492024-11-21T19:33:17.909ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
5502024-11-21T19:33:17.909ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
5512024-11-21T19:33:17.910ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5522024-11-21T19:33:17.912ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5532024-11-21T19:33:17.914ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
5542024-11-21T19:33:17.914ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
5552024-11-21T19:33:17.915ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
5562024-11-21T19:33:17.915ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 86.932222ms iteration = 1 status = {}
5572024-11-21T19:33:17.916ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 740, tv_nsec: 796958922 } })
5582024-11-21T19:33:17.916ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
5592024-11-21T19:33:17.916ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
5602024-11-21T19:33:17.917ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
5612024-11-21T19:33:17.917ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
5622024-11-21T19:33:17.917ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5632024-11-21T19:33:17.921ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
5642024-11-21T19:33:17.922ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
5652024-11-21T19:33:17.924ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
5662024-11-21T19:33:17.924ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
5672024-11-21T19:33:17.928ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
5682024-11-21T19:33:17.929ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
5692024-11-21T19:33:17.929ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
5702024-11-21T19:33:17.929ZWARNnexus (BackgroundTasks): inventory collection is None background_task = service_zone_nat_tracker file = nexus/src/app/background/tasks/sync_service_zone_nat.rs:95
5712024-11-21T19:33:17.929ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
5722024-11-21T19:33:17.929ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 101.177472ms iteration = 1 status = {"error":"inventory collection is None"}
5732024-11-21T19:33:17.930ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5742024-11-21T19:33:17.930ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5752024-11-21T19:33:17.931ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 102.778123ms iteration = 1 status = {"errors":[],"records_set_to_done":[]}
5762024-11-21T19:33:17.931ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5772024-11-21T19:33:17.931ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
5782024-11-21T19:33:17.931ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
5792024-11-21T19:33:17.931ZINFOnexus (SagaRecoverer): listed in-progress sagas count = 0 file = nexus/src/app/background/tasks/saga_recovery.rs:438
5802024-11-21T19:33:17.931ZDEBGnexus (BackgroundTasks): activation complete background_task = saga_recovery elapsed = 103.076527ms iteration = 1 status = {"last_pass":{"Success":{"nfailed":0,"nfound":0,"nrecovered":0,"nremoved":0,"nskipped":0}},"ntotal_failures":0,"ntotal_finished":0,"ntotal_recovered":0,"ntotal_sec_errors_bad_state":0,"ntotal_sec_errors_missing":0,"ntotal_started":0,"recent_failures":[],"recent_recoveries":[]}
5812024-11-21T19:33:17.931ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
5822024-11-21T19:33:17.931ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "09804941-a426-4ef8-9764-66d07ded46f9", "content-length": "26", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
5832024-11-21T19:33:17.931ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0 total = 0
5842024-11-21T19:33:17.932ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:171 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0 vmms_found = 0
5852024-11-21T19:33:17.932ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 103.501723ms iteration = 1 status = {"errors":[],"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0,"vmms_found":0}
5862024-11-21T19:33:17.932ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
5872024-11-21T19:33:17.933ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
5882024-11-21T19:33:17.934ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
5892024-11-21T19:33:17.934ZDEBGnexus (BackgroundTasks): authorize result action = Modify actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Inventory result = Ok(())
5902024-11-21T19:33:17.934ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {} }
5912024-11-21T19:33:17.935ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Database result = Ok(())
5922024-11-21T19:33:17.935ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
5932024-11-21T19:33:17.935ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
5942024-11-21T19:33:17.935ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
5952024-11-21T19:33:17.935ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
5962024-11-21T19:33:17.936ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
5972024-11-21T19:33:17.936ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
5982024-11-21T19:33:17.936ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
5992024-11-21T19:33:17.936ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
6002024-11-21T19:33:17.936ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
6012024-11-21T19:33:17.937ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
6022024-11-21T19:33:17.939ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
6032024-11-21T19:33:17.939ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
6042024-11-21T19:33:17.939ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
6052024-11-21T19:33:17.939ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
6062024-11-21T19:33:17.939ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
6072024-11-21T19:33:17.939ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
6082024-11-21T19:33:17.940ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
6092024-11-21T19:33:17.941ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
6102024-11-21T19:33:17.941ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
6112024-11-21T19:33:17.941ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
6122024-11-21T19:33:17.943ZDEBGnexus (ServiceBalancer): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Rack { parent: Fleet, key: 6f50eaaf-6509-4592-a570-21594f7b8789, lookup_type: ById(6f50eaaf-6509-4592-a570-21594f7b8789) } result = Ok(())
6132024-11-21T19:33:17.943ZINFOnexus: Still waiting for rack initialization: Rack { identity: RackIdentity { id: 6f50eaaf-6509-4592-a570-21594f7b8789, time_created: 2024-11-21T19:33:17.521183Z, time_modified: 2024-11-21T19:33:17.521183Z }, initialized: false, tuf_base_url: None, rack_subnet: Some(V6(Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 })) } file = nexus/src/app/rack.rs:768
6142024-11-21T19:33:17.944ZINFOnexus (BackgroundTasks): Physical Disk Adoption: Rack not yet initialized background_task = physical_disk_adoption file = nexus/src/app/background/tasks/physical_disk_adoption.rs:79 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
6152024-11-21T19:33:17.944ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 115.967109ms iteration = 1 status = {"error":"rack not yet initialized: 6f50eaaf-6509-4592-a570-21594f7b8789"}
6162024-11-21T19:33:17.952ZDEBGnexus (BackgroundTasks): inventory_prune_one: nothing eligible for removal (too few) background_task = inventory_collection candidates = []
6172024-11-21T19:33:17.953ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
6182024-11-21T19:33:17.953ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
6192024-11-21T19:33:17.953ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _mgs._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_mgs._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_mgs._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12225, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 740, tv_nsec: 834375312 } })
6202024-11-21T19:33:17.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
6212024-11-21T19:33:17.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
6222024-11-21T19:33:17.960ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
6232024-11-21T19:33:17.961ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
6242024-11-21T19:33:17.971ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 143.625089ms iteration = 1 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
6252024-11-21T19:33:17.973ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 144.45416ms iteration = 1 status = {"errors":[],"garbage_collect_requested":[]}
6262024-11-21T19:33:17.985ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
6272024-11-21T19:33:17.985ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
6282024-11-21T19:33:17.994ZWARNnexus (BackgroundTasks): failed to read DNS config background_task = dns_config_internal dns_group = internal error = Internal Error: expected exactly one latest version for DNS group Internal, found 0 file = nexus/src/app/background/tasks/dns_config.rs:72
6292024-11-21T19:33:17.994ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 166.069959ms iteration = 1 status = {"error":"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group Internal, found 0"}
6302024-11-21T19:33:17.996ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 167.451137ms iteration = 1 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
6312024-11-21T19:33:17.999ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
6322024-11-21T19:33:17.999ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
6332024-11-21T19:33:18.001ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
6342024-11-21T19:33:18.002ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
6352024-11-21T19:33:18.002ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 173.673921ms iteration = 1 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
6362024-11-21T19:33:18.002ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
6372024-11-21T19:33:18.005ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
6382024-11-21T19:33:18.005ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
6392024-11-21T19:33:18.005ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
6402024-11-21T19:33:18.007ZWARNnexus (BackgroundTasks): failed to read DNS config background_task = dns_config_external dns_group = external error = Internal Error: expected exactly one latest version for DNS group External, found 0 file = nexus/src/app/background/tasks/dns_config.rs:72
6412024-11-21T19:33:18.007ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 178.856469ms iteration = 1 status = {"error":"failed to read DNS config: Internal Error: expected exactly one latest version for DNS group External, found 0"}
6422024-11-21T19:33:18.008ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
6432024-11-21T19:33:18.008ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
6442024-11-21T19:33:18.009ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "2fa498f4-4bf5-44cc-ac38-f3799401d2fa", "content-length": "26", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
6452024-11-21T19:33:18.009ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
6462024-11-21T19:33:18.010ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
6472024-11-21T19:33:18.013ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
6482024-11-21T19:33:18.013ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 185.266776ms iteration = 1 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
6492024-11-21T19:33:18.020ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
6502024-11-21T19:33:18.021ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "78e54c52-6205-44e3-9603-405e52e873ad", "content-length": "1", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
6512024-11-21T19:33:18.023ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
6522024-11-21T19:33:18.023ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
6532024-11-21T19:33:18.024ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 195.568523ms iteration = 1 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
6542024-11-21T19:33:18.026ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 197.940995ms iteration = 1 status = null
6552024-11-21T19:33:18.028ZWARNnexus (BackgroundTasks): failed to read Silo/DNS/TLS configuration background_task = external_endpoints error = Internal Error: expected at least one external DNS zone file = nexus/src/app/background/tasks/external_endpoints.rs:48
6562024-11-21T19:33:18.028ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 200.159245ms iteration = 1 status = {"error":"failed to read Silo/DNS/TLS configuration: Internal Error: expected at least one external DNS zone"}
6572024-11-21T19:33:18.028ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
6582024-11-21T19:33:18.030ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
6592024-11-21T19:33:18.030ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
6602024-11-21T19:33:18.030ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 202.040589ms iteration = 1 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
6612024-11-21T19:33:18.032ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
6622024-11-21T19:33:18.032ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
6632024-11-21T19:33:18.032ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
6642024-11-21T19:33:18.034ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
6652024-11-21T19:33:18.035ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
6662024-11-21T19:33:18.036ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
6672024-11-21T19:33:18.036ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
6682024-11-21T19:33:18.037ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
6692024-11-21T19:33:18.040ZDEBGnexus (BackgroundTasks): begin collection background_task = inventory_collection
6702024-11-21T19:33:18.040ZDEBGnexus (BackgroundTasks): begin collection from MGS background_task = inventory_collection mgs_url = http://[fd00:1122:3344:101::2]:12225
6712024-11-21T19:33:18.040ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/ignition
6722024-11-21T19:33:18.041ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/ignition", status: 200, headers: {"content-type": "application/json", "x-request-id": "911fe03e-80dd-4f1f-9639-92e7b8bcfac8", "content-length": "476", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
6732024-11-21T19:33:18.041ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
6742024-11-21T19:33:18.041ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
6752024-11-21T19:33:18.041ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
6762024-11-21T19:33:18.043ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0
6772024-11-21T19:33:18.044ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0", status: 200, headers: {"content-type": "application/json", "x-request-id": "3618556c-e12b-4562-8674-8bbf6a6ac6b9", "content-length": "734", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
6782024-11-21T19:33:18.045ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 216.524644ms iteration = 1 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
6792024-11-21T19:33:18.045ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/sp/caboose?firmware_slot=0
6802024-11-21T19:33:18.045ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/sp/caboose?firmware_slot=0", status: 200, headers: {"content-type": "application/json", "x-request-id": "384f7521-2350-411c-aa49-5558e95e506b", "content-length": "111", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
6812024-11-21T19:33:18.046ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/sp/caboose?firmware_slot=1
6822024-11-21T19:33:18.046ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/sp/caboose?firmware_slot=1", status: 200, headers: {"content-type": "application/json", "x-request-id": "8bf031db-c9d3-4cba-8d3e-d6f2977566be", "content-length": "111", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
6832024-11-21T19:33:18.047ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/caboose?firmware_slot=0
6842024-11-21T19:33:18.047ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/caboose?firmware_slot=0", status: 200, headers: {"content-type": "application/json", "x-request-id": "2c949e73-90c5-4971-92ba-c9991b4188bc", "content-length": "167", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
6852024-11-21T19:33:18.047ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/caboose?firmware_slot=1
6862024-11-21T19:33:18.048ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/caboose?firmware_slot=1", status: 200, headers: {"content-type": "application/json", "x-request-id": "7bc74bc9-bf7e-4db1-85e1-40877bb8231d", "content-length": "167", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
6872024-11-21T19:33:18.048ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/stage0/caboose?firmware_slot=0
6882024-11-21T19:33:18.048ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/stage0/caboose?firmware_slot=0", status: 200, headers: {"content-type": "application/json", "x-request-id": "84569a17-20eb-4d4d-bf72-e695d15b3e01", "content-length": "175", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
6892024-11-21T19:33:18.048ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/stage0/caboose?firmware_slot=1
6902024-11-21T19:33:18.048ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/stage0/caboose?firmware_slot=1", status: 200, headers: {"content-type": "application/json", "x-request-id": "69128188-1840-41f8-80da-90e3511a58d0", "content-length": "175", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
6912024-11-21T19:33:18.048ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/cmpa
6922024-11-21T19:33:18.048ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/cmpa", status: 200, headers: {"content-type": "application/json", "x-request-id": "3cbc0082-8c4d-4546-98fb-e0c7df5c85a0", "content-length": "702", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
6932024-11-21T19:33:18.048ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = Some(Body) gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/cfpa
6942024-11-21T19:33:18.049ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/cfpa", status: 200, headers: {"content-type": "application/json", "x-request-id": "0e82ba94-117b-4a4f-b0d3-9a778f6f0a1a", "content-length": "727", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
6952024-11-21T19:33:18.049ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = Some(Body) gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/cfpa
6962024-11-21T19:33:18.049ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/cfpa", status: 200, headers: {"content-type": "application/json", "x-request-id": "d0e90b11-d5b4-49c2-abd7-ad7624e1658f", "content-length": "729", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
6972024-11-21T19:33:18.049ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = Some(Body) gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/cfpa
6982024-11-21T19:33:18.049ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/cfpa", status: 200, headers: {"content-type": "application/json", "x-request-id": "17127886-1c48-4898-8840-9a18fa7a7516", "content-length": "728", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
6992024-11-21T19:33:18.049ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0
7002024-11-21T19:33:18.049ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0", status: 200, headers: {"content-type": "application/json", "x-request-id": "b0cb508d-bcfb-47db-b8cd-3009ac7b42f0", "content-length": "722", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
7012024-11-21T19:33:18.049ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/sp/caboose?firmware_slot=0
7022024-11-21T19:33:18.049ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/sp/caboose?firmware_slot=0", status: 200, headers: {"content-type": "application/json", "x-request-id": "895dfe9b-c07a-4e8a-a95a-e727fb36330d", "content-length": "109", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
7032024-11-21T19:33:18.049ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/sp/caboose?firmware_slot=1
7042024-11-21T19:33:18.050ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/sp/caboose?firmware_slot=1", status: 200, headers: {"content-type": "application/json", "x-request-id": "7566f6c3-cca8-415c-8a11-68151d374361", "content-length": "109", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
7052024-11-21T19:33:18.050ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/caboose?firmware_slot=0
7062024-11-21T19:33:18.050ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/caboose?firmware_slot=0", status: 200, headers: {"content-type": "application/json", "x-request-id": "408ec5ec-64ad-48df-8b99-e38ee27f5234", "content-length": "169", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
7072024-11-21T19:33:18.050ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/caboose?firmware_slot=1
7082024-11-21T19:33:18.050ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/caboose?firmware_slot=1", status: 200, headers: {"content-type": "application/json", "x-request-id": "9a194cbd-4ab0-43ab-b775-3d87fbec350e", "content-length": "169", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
7092024-11-21T19:33:18.050ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/stage0/caboose?firmware_slot=0
7102024-11-21T19:33:18.050ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/stage0/caboose?firmware_slot=0", status: 200, headers: {"content-type": "application/json", "x-request-id": "68569f7a-94f1-4c91-af00-7f290eeb5136", "content-length": "178", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
7112024-11-21T19:33:18.050ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/stage0/caboose?firmware_slot=1
7122024-11-21T19:33:18.051ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/stage0/caboose?firmware_slot=1", status: 200, headers: {"content-type": "application/json", "x-request-id": "bd7638b5-fec5-49b8-b68e-beefc39c70ed", "content-length": "178", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
7132024-11-21T19:33:18.051ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/cmpa
7142024-11-21T19:33:18.051ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/cmpa", status: 200, headers: {"content-type": "application/json", "x-request-id": "2cfcd7f0-7278-4440-b1db-c811d986fa47", "content-length": "702", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
7152024-11-21T19:33:18.051ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = Some(Body) gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/cfpa
7162024-11-21T19:33:18.051ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/cfpa", status: 200, headers: {"content-type": "application/json", "x-request-id": "32157002-0067-43a8-a94b-c32fce904d46", "content-length": "727", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
7172024-11-21T19:33:18.051ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = Some(Body) gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/cfpa
7182024-11-21T19:33:18.051ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/cfpa", status: 200, headers: {"content-type": "application/json", "x-request-id": "b6554b1a-f4a7-4962-8062-0cc5a41afeec", "content-length": "729", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
7192024-11-21T19:33:18.051ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = Some(Body) gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/cfpa
7202024-11-21T19:33:18.052ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/cfpa", status: 200, headers: {"content-type": "application/json", "x-request-id": "761024e2-ecd0-4175-98f0-729b5fac4bc9", "content-length": "728", "date": "Thu, 21 Nov 2024 19:33:17 GMT"} })
7212024-11-21T19:33:18.052ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {} }
7222024-11-21T19:33:18.052ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Database result = Ok(())
7232024-11-21T19:33:18.054ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7242024-11-21T19:33:18.056ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Fleet result = Ok(())
7252024-11-21T19:33:18.056ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {} }
7262024-11-21T19:33:18.056ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Database result = Ok(())
7272024-11-21T19:33:18.062ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 233.951346ms iteration = 1 status = {}
7282024-11-21T19:33:18.081ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7292024-11-21T19:33:18.081ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
7302024-11-21T19:33:18.081ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
7312024-11-21T19:33:18.081ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
7322024-11-21T19:33:18.083ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7332024-11-21T19:33:18.083ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
7342024-11-21T19:33:18.085ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7352024-11-21T19:33:18.085ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
7362024-11-21T19:33:18.087ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7372024-11-21T19:33:18.087ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
7382024-11-21T19:33:18.088ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7392024-11-21T19:33:18.090ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Fleet result = Ok(())
7402024-11-21T19:33:18.090ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {} }
7412024-11-21T19:33:18.090ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Database result = Ok(())
7422024-11-21T19:33:18.093ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7432024-11-21T19:33:18.093ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
7442024-11-21T19:33:18.104ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 276.481074ms iteration = 1 status = {}
7452024-11-21T19:33:18.117ZWARNnexus (BackgroundTasks): instance reincarnation completed with errors background_task = instance_reincarnation file = nexus/src/app/background/tasks/instance_reincarnation.rs:103 instances_changed_state = 0 instances_found = 0 instances_reincarnated = 0 query_errors = 0 restart_errors = 0
7462024-11-21T19:33:18.117ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_reincarnation elapsed = 289.424147ms iteration = 1 status = {"changed_state":[],"disabled":false,"errors":[],"instances_found":{"Failed":0,"SagaUnwound":0},"instances_reincarnated":[],"restart_errors":[]}
7472024-11-21T19:33:18.142ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7482024-11-21T19:33:18.143ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
7492024-11-21T19:33:18.144ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7502024-11-21T19:33:18.147ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
7512024-11-21T19:33:18.147ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7522024-11-21T19:33:18.147ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
7532024-11-21T19:33:18.149ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7542024-11-21T19:33:18.149ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
7552024-11-21T19:33:18.151ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7562024-11-21T19:33:18.152ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
7572024-11-21T19:33:18.154ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7582024-11-21T19:33:18.154ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
7592024-11-21T19:33:18.156ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7602024-11-21T19:33:18.156ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
7612024-11-21T19:33:18.166ZDEBGnexus (BackgroundTasks): begin collecting all keepers background_task = inventory_collection nkeeper_admin_clients = 0
7622024-11-21T19:33:18.166ZDEBGnexus (BackgroundTasks): end collecting all keepers background_task = inventory_collection nkeeper_admin_clients = 0
7632024-11-21T19:33:18.166ZDEBGnexus (BackgroundTasks): finished collection background_task = inventory_collection
7642024-11-21T19:33:18.167ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {} }
7652024-11-21T19:33:18.167ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Database result = Ok(())
7662024-11-21T19:33:18.169ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7672024-11-21T19:33:18.171ZDEBGnexus (BackgroundTasks): authorize result action = Modify actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Inventory result = Ok(())
7682024-11-21T19:33:18.171ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {} }
7692024-11-21T19:33:18.171ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Database result = Ok(())
7702024-11-21T19:33:18.193ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7712024-11-21T19:33:18.193ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
7722024-11-21T19:33:18.194ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7732024-11-21T19:33:18.194ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
7742024-11-21T19:33:18.196ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7752024-11-21T19:33:18.200ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000001, lookup_type: ByName("external-dns") } result = Ok(())
7762024-11-21T19:33:18.200ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7772024-11-21T19:33:18.200ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
7782024-11-21T19:33:18.201ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7792024-11-21T19:33:18.201ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
7802024-11-21T19:33:18.203ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7812024-11-21T19:33:18.203ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
7822024-11-21T19:33:18.204ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
7832024-11-21T19:33:18.205ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
7842024-11-21T19:33:18.206ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7852024-11-21T19:33:18.208ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
7862024-11-21T19:33:18.208ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
7872024-11-21T19:33:18.208ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
7882024-11-21T19:33:18.209ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7892024-11-21T19:33:18.211ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
7902024-11-21T19:33:18.211ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
7912024-11-21T19:33:18.211ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
7922024-11-21T19:33:18.233ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
7932024-11-21T19:33:18.237ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000001, lookup_type: ByName("external-dns") } result = Ok(())
7942024-11-21T19:33:18.237ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7952024-11-21T19:33:18.237ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
7962024-11-21T19:33:18.255ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7972024-11-21T19:33:18.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
7982024-11-21T19:33:18.257ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
7992024-11-21T19:33:18.257ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
8002024-11-21T19:33:18.259ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
8012024-11-21T19:33:18.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
8022024-11-21T19:33:18.262ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
8032024-11-21T19:33:18.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
8042024-11-21T19:33:18.264ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
8052024-11-21T19:33:18.265ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
8062024-11-21T19:33:18.266ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
8072024-11-21T19:33:18.266ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
8082024-11-21T19:33:18.267ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
8092024-11-21T19:33:18.268ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
8102024-11-21T19:33:18.269ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8112024-11-21T19:33:18.272ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000002, lookup_type: ByName("nexus") } result = Ok(())
8122024-11-21T19:33:18.272ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
8132024-11-21T19:33:18.272ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
8142024-11-21T19:33:18.274ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
8152024-11-21T19:33:18.274ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
8162024-11-21T19:33:18.277ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
8172024-11-21T19:33:18.277ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
8182024-11-21T19:33:18.277ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 449.467036ms iteration = 1 status = {}
8192024-11-21T19:33:18.278ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8202024-11-21T19:33:18.282ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000002, lookup_type: ByName("nexus") } result = Ok(())
8212024-11-21T19:33:18.282ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
8222024-11-21T19:33:18.282ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
8232024-11-21T19:33:18.288ZDEBGnexus (service-firewall-rule-progator): resolved names for firewall rules background_task = service_firewall_rule_propagation instance_interfaces = {} subnet_interfaces = {} subnet_networks = {Name("external-dns"): [V4(Ipv4Network { addr: 172.30.1.0, prefix: 24 }), V6(Ipv6Network { addr: fd77:e9d2:9cd9:1::, prefix: 64 })], Name("nexus"): [V4(Ipv4Network { addr: 172.30.2.0, prefix: 24 }), V6(Ipv6Network { addr: fd77:e9d2:9cd9:2::, prefix: 64 })]} vpc_interfaces = {}
8242024-11-21T19:33:18.289ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
8252024-11-21T19:33:18.290ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
8262024-11-21T19:33:18.291ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8272024-11-21T19:33:18.293ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Fleet result = Ok(())
8282024-11-21T19:33:18.293ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
8292024-11-21T19:33:18.293ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
8302024-11-21T19:33:18.312ZDEBGnexus (service-firewall-rule-progator): Allowlist for user-facing services is set to allow any inbound traffic. Existing VPC firewall rules will not be modified. background_task = service_firewall_rule_propagation
8312024-11-21T19:33:18.312ZDEBGnexus (service-firewall-rule-progator): resolved firewall rules for sled agents background_task = service_firewall_rule_propagation sled_agent_rules = []
8322024-11-21T19:33:18.312ZDEBGnexus (service-firewall-rule-progator): resolved 0 rules for sleds background_task = service_firewall_rule_propagation
8332024-11-21T19:33:18.437ZDEBGnexus (service-firewall-rule-progator): resolved sleds for vpc oxide-services background_task = service_firewall_rule_propagation vpc_to_sled = []
8342024-11-21T19:33:18.437ZDEBGnexus (service-firewall-rule-progator): sending firewall rules to sled agents background_task = service_firewall_rule_propagation
8352024-11-21T19:33:18.437ZINFOnexus (service-firewall-rule-progator): updated firewall rules on 0 sleds background_task = service_firewall_rule_propagation file = nexus/networking/src/firewall_rules.rs:496
8362024-11-21T19:33:18.438ZDEBGnexus (service-firewall-rule-progator): successfully propagated service firewall rules background_task = service_firewall_rule_propagation elapsed = 606.816038ms
8372024-11-21T19:33:18.438ZDEBGnexus (BackgroundTasks): activation complete background_task = service_firewall_rule_propagation elapsed = 609.21798ms iteration = 1 status = {}
8382024-11-21T19:33:18.466ZINFOnexus (BackgroundTasks): inserted inventory collection background_task = inventory_collection collection_id = 0322ca29-70af-440b-84f5-66321d76596e file = nexus/db-queries/src/db/datastore/inventory.rs:1000
8392024-11-21T19:33:18.466ZDEBGnexus (BackgroundTasks): inventory collection complete background_task = inventory_collection collection_id = 0322ca29-70af-440b-84f5-66321d76596e time_started = 2024-11-21 19:33:18.039748 UTC
8402024-11-21T19:33:18.466ZDEBGnexus (BackgroundTasks): activation complete background_task = inventory_collection elapsed = 638.156337ms iteration = 1 status = {"collection_id":"0322ca29-70af-440b-84f5-66321d76596e","time_done":"2024-11-21 19:33:18.166604 UTC","time_started":"2024-11-21 19:33:18.039748 UTC"}
8412024-11-21T19:33:18.466ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 2 reason = Dependency
8422024-11-21T19:33:18.466ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
8432024-11-21T19:33:18.466ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
8442024-11-21T19:33:18.468ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8452024-11-21T19:33:18.470ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
8462024-11-21T19:33:18.470ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
8472024-11-21T19:33:18.470ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
8482024-11-21T19:33:18.471ZINFOnexus (BackgroundTasks): Physical Disk Adoption: Rack not yet initialized background_task = physical_disk_adoption file = nexus/src/app/background/tasks/physical_disk_adoption.rs:79 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
8492024-11-21T19:33:18.471ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 5.174629ms iteration = 2 status = {"error":"rack not yet initialized: 6f50eaaf-6509-4592-a570-21594f7b8789"}
8502024-11-21T19:33:19.944ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
8512024-11-21T19:33:19.945ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
8522024-11-21T19:33:19.946ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
8532024-11-21T19:33:19.946ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
8542024-11-21T19:33:19.948ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8552024-11-21T19:33:19.950ZDEBGnexus (ServiceBalancer): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Rack { parent: Fleet, key: 6f50eaaf-6509-4592-a570-21594f7b8789, lookup_type: ById(6f50eaaf-6509-4592-a570-21594f7b8789) } result = Ok(())
8562024-11-21T19:33:19.951ZINFOnexus: Still waiting for rack initialization: Rack { identity: RackIdentity { id: 6f50eaaf-6509-4592-a570-21594f7b8789, time_created: 2024-11-21T19:33:17.521183Z, time_modified: 2024-11-21T19:33:17.521183Z }, initialized: false, tuf_base_url: None, rack_subnet: Some(V6(Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 })) } file = nexus/src/app/rack.rs:768
8572024-11-21T19:33:21.951ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
8582024-11-21T19:33:21.951ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
8592024-11-21T19:33:21.953ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
8602024-11-21T19:33:21.953ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
8612024-11-21T19:33:21.957ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8622024-11-21T19:33:21.960ZDEBGnexus (ServiceBalancer): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Rack { parent: Fleet, key: 6f50eaaf-6509-4592-a570-21594f7b8789, lookup_type: ById(6f50eaaf-6509-4592-a570-21594f7b8789) } result = Ok(())
8632024-11-21T19:33:21.960ZINFOnexus: Still waiting for rack initialization: Rack { identity: RackIdentity { id: 6f50eaaf-6509-4592-a570-21594f7b8789, time_created: 2024-11-21T19:33:17.521183Z, time_modified: 2024-11-21T19:33:17.521183Z }, initialized: false, tuf_base_url: None, rack_subnet: Some(V6(Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 })) } file = nexus/src/app/rack.rs:768
8642024-11-21T19:33:23.961ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
8652024-11-21T19:33:23.961ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
8662024-11-21T19:33:23.963ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
8672024-11-21T19:33:23.963ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
8682024-11-21T19:33:23.965ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8692024-11-21T19:33:23.968ZDEBGnexus (ServiceBalancer): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Rack { parent: Fleet, key: 6f50eaaf-6509-4592-a570-21594f7b8789, lookup_type: ById(6f50eaaf-6509-4592-a570-21594f7b8789) } result = Ok(())
8702024-11-21T19:33:23.968ZINFOnexus: Still waiting for rack initialization: Rack { identity: RackIdentity { id: 6f50eaaf-6509-4592-a570-21594f7b8789, time_created: 2024-11-21T19:33:17.521183Z, time_modified: 2024-11-21T19:33:17.521183Z }, initialized: false, tuf_base_url: None, rack_subnet: Some(V6(Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 })) } file = nexus/src/app/rack.rs:768
8712024-11-21T19:33:25.968ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
8722024-11-21T19:33:25.968ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
8732024-11-21T19:33:25.971ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
8742024-11-21T19:33:25.971ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
8752024-11-21T19:33:25.973ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8762024-11-21T19:33:25.975ZDEBGnexus (ServiceBalancer): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Rack { parent: Fleet, key: 6f50eaaf-6509-4592-a570-21594f7b8789, lookup_type: ById(6f50eaaf-6509-4592-a570-21594f7b8789) } result = Ok(())
8772024-11-21T19:33:25.975ZINFOnexus: Still waiting for rack initialization: Rack { identity: RackIdentity { id: 6f50eaaf-6509-4592-a570-21594f7b8789, time_created: 2024-11-21T19:33:17.521183Z, time_modified: 2024-11-21T19:33:17.521183Z }, initialized: false, tuf_base_url: None, rack_subnet: Some(V6(Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 })) } file = nexus/src/app/rack.rs:768
8782024-11-21T19:33:27.827ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 2 reason = Timeout
8792024-11-21T19:33:27.827ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
8802024-11-21T19:33:27.828ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
8812024-11-21T19:33:27.828ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 2 reason = Timeout
8822024-11-21T19:33:27.828ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
8832024-11-21T19:33:27.828ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
8842024-11-21T19:33:27.829ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8852024-11-21T19:33:27.832ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
8862024-11-21T19:33:27.832ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
8872024-11-21T19:33:27.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
8882024-11-21T19:33:27.834ZWARNnexus (BackgroundTasks): failed to read target blueprint background_task = blueprint_loader error = Internal Error: no target blueprint set file = nexus/src/app/background/tasks/blueprint_load.rs:69
8892024-11-21T19:33:27.834ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 6.916012ms iteration = 2 status = {"error":"failed to read target blueprint: Internal Error: no target blueprint set"}
8902024-11-21T19:33:27.868ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
8912024-11-21T19:33:27.868ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
8922024-11-21T19:33:27.901ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 73.025835ms iteration = 2 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
8932024-11-21T19:33:27.977ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
8942024-11-21T19:33:27.977ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
8952024-11-21T19:33:27.978ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
8962024-11-21T19:33:27.978ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
8972024-11-21T19:33:27.980ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
8982024-11-21T19:33:27.983ZDEBGnexus (ServiceBalancer): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Rack { parent: Fleet, key: 6f50eaaf-6509-4592-a570-21594f7b8789, lookup_type: ById(6f50eaaf-6509-4592-a570-21594f7b8789) } result = Ok(())
8992024-11-21T19:33:27.983ZINFOnexus: Still waiting for rack initialization: Rack { identity: RackIdentity { id: 6f50eaaf-6509-4592-a570-21594f7b8789, time_created: 2024-11-21T19:33:17.521183Z, time_modified: 2024-11-21T19:33:17.521183Z }, initialized: false, tuf_base_url: None, rack_subnet: Some(V6(Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 })) } file = nexus/src/app/rack.rs:768
9002024-11-21T19:33:29.984ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
9012024-11-21T19:33:29.984ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
9022024-11-21T19:33:29.985ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
9032024-11-21T19:33:29.985ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
9042024-11-21T19:33:29.987ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9052024-11-21T19:33:29.990ZDEBGnexus (ServiceBalancer): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Rack { parent: Fleet, key: 6f50eaaf-6509-4592-a570-21594f7b8789, lookup_type: ById(6f50eaaf-6509-4592-a570-21594f7b8789) } result = Ok(())
9062024-11-21T19:33:29.990ZINFOnexus: Still waiting for rack initialization: Rack { identity: RackIdentity { id: 6f50eaaf-6509-4592-a570-21594f7b8789, time_created: 2024-11-21T19:33:17.521183Z, time_modified: 2024-11-21T19:33:17.521183Z }, initialized: false, tuf_base_url: None, rack_subnet: Some(V6(Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 })) } file = nexus/src/app/rack.rs:768
9072024-11-21T19:33:31.990ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
9082024-11-21T19:33:31.990ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
9092024-11-21T19:33:31.995ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
9102024-11-21T19:33:31.995ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
9112024-11-21T19:33:31.998ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9122024-11-21T19:33:32.001ZDEBGnexus (ServiceBalancer): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Rack { parent: Fleet, key: 6f50eaaf-6509-4592-a570-21594f7b8789, lookup_type: ById(6f50eaaf-6509-4592-a570-21594f7b8789) } result = Ok(())
9132024-11-21T19:33:32.001ZINFOnexus: Still waiting for rack initialization: Rack { identity: RackIdentity { id: 6f50eaaf-6509-4592-a570-21594f7b8789, time_created: 2024-11-21T19:33:17.521183Z, time_modified: 2024-11-21T19:33:17.521183Z }, initialized: false, tuf_base_url: None, rack_subnet: Some(V6(Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 })) } file = nexus/src/app/rack.rs:768
9142024-11-21T19:33:34.002ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
9152024-11-21T19:33:34.002ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
9162024-11-21T19:33:34.004ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {} }
9172024-11-21T19:33:34.004ZDEBGnexus (ServiceBalancer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Database result = Ok(())
9182024-11-21T19:33:34.006ZDEBGnexus (ServiceBalancer): roles roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9192024-11-21T19:33:34.009ZDEBGnexus (ServiceBalancer): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-00000000bac3, .. }) resource = Rack { parent: Fleet, key: 6f50eaaf-6509-4592-a570-21594f7b8789, lookup_type: ById(6f50eaaf-6509-4592-a570-21594f7b8789) } result = Ok(())
9202024-11-21T19:33:34.009ZINFOnexus: Rack initialized file = nexus/src/app/rack.rs:765
9212024-11-21T19:33:34.009ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9222024-11-21T19:33:34.009ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9232024-11-21T19:33:34.014ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9242024-11-21T19:33:34.014ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9252024-11-21T19:33:34.019ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9262024-11-21T19:33:34.019ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9272024-11-21T19:33:34.021ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9282024-11-21T19:33:34.021ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9292024-11-21T19:33:34.023ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9302024-11-21T19:33:34.023ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9312024-11-21T19:33:34.024ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9322024-11-21T19:33:34.025ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9332024-11-21T19:33:34.026ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9342024-11-21T19:33:34.029ZDEBGnexus (InternalApi): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
9352024-11-21T19:33:34.029ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9362024-11-21T19:33:34.029ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9372024-11-21T19:33:34.031ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9382024-11-21T19:33:34.031ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9392024-11-21T19:33:34.032ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9402024-11-21T19:33:34.033ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9412024-11-21T19:33:34.034ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9422024-11-21T19:33:34.037ZDEBGnexus (InternalApi): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
9432024-11-21T19:33:34.037ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9442024-11-21T19:33:34.037ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9452024-11-21T19:33:34.039ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9462024-11-21T19:33:34.039ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9472024-11-21T19:33:34.041ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9482024-11-21T19:33:34.041ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9492024-11-21T19:33:34.042ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9502024-11-21T19:33:34.042ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9512024-11-21T19:33:34.044ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9522024-11-21T19:33:34.044ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9532024-11-21T19:33:34.046ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9542024-11-21T19:33:34.046ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9552024-11-21T19:33:34.047ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9562024-11-21T19:33:34.047ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9572024-11-21T19:33:34.048ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9582024-11-21T19:33:34.051ZDEBGnexus (InternalApi): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
9592024-11-21T19:33:34.055ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9602024-11-21T19:33:34.055ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9612024-11-21T19:33:34.057ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9622024-11-21T19:33:34.057ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9632024-11-21T19:33:34.058ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9642024-11-21T19:33:34.058ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9652024-11-21T19:33:34.060ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9662024-11-21T19:33:34.060ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9672024-11-21T19:33:34.062ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9682024-11-21T19:33:34.062ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9692024-11-21T19:33:34.064ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9702024-11-21T19:33:34.064ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9712024-11-21T19:33:34.066ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9722024-11-21T19:33:34.066ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9732024-11-21T19:33:34.067ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9742024-11-21T19:33:34.070ZDEBGnexus (InternalApi): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000001, lookup_type: ByName("external-dns") } result = Ok(())
9752024-11-21T19:33:34.070ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9762024-11-21T19:33:34.070ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9772024-11-21T19:33:34.072ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9782024-11-21T19:33:34.073ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9792024-11-21T19:33:34.074ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9802024-11-21T19:33:34.074ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9812024-11-21T19:33:34.075ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
9822024-11-21T19:33:34.078ZDEBGnexus (InternalApi): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000001, lookup_type: ByName("external-dns") } result = Ok(())
9832024-11-21T19:33:34.078ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9842024-11-21T19:33:34.078ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9852024-11-21T19:33:34.087ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9862024-11-21T19:33:34.087ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9872024-11-21T19:33:34.089ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9882024-11-21T19:33:34.089ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9892024-11-21T19:33:34.091ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9902024-11-21T19:33:34.091ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9912024-11-21T19:33:34.093ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9922024-11-21T19:33:34.093ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9932024-11-21T19:33:34.095ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9942024-11-21T19:33:34.095ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9952024-11-21T19:33:34.096ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9962024-11-21T19:33:34.097ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9972024-11-21T19:33:34.098ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
9982024-11-21T19:33:34.098ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
9992024-11-21T19:33:34.099ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10002024-11-21T19:33:34.103ZDEBGnexus (InternalApi): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000002, lookup_type: ByName("nexus") } result = Ok(())
10012024-11-21T19:33:34.103ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
10022024-11-21T19:33:34.103ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
10032024-11-21T19:33:34.104ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
10042024-11-21T19:33:34.104ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
10052024-11-21T19:33:34.106ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
10062024-11-21T19:33:34.106ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
10072024-11-21T19:33:34.107ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10082024-11-21T19:33:34.110ZDEBGnexus (InternalApi): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000002, lookup_type: ByName("nexus") } result = Ok(())
10092024-11-21T19:33:34.110ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
10102024-11-21T19:33:34.110ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
10112024-11-21T19:33:34.119ZDEBGnexus (InternalApi): resolved names for firewall rules instance_interfaces = {} subnet_interfaces = {Name("external-dns"): [NetworkInterface { id: 4c211da9-c36c-41bc-8274-9aa6420c20e6, kind: Service { id: 90b200e8-d1ee-4e0d-975c-64628c5532d2 }, name: Name("external-dns-90b200e8-d1ee-4e0d-975c-64628c5532d2"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 185])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b69bf686-c443-40c6-b854-c2ff6e5e107a, kind: Service { id: 4e669ff4-6dde-4cd2-8491-623460b56d8d }, name: Name("external-dns-4e669ff4-6dde-4cd2-8491-623460b56d8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 188])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], Name("nexus"): [NetworkInterface { id: 03ccecdf-44ea-4815-8920-5ebe8d7bf7ff, kind: Service { id: 915e11b8-d7cb-4730-9906-7980540813dc }, name: Name("nexus-915e11b8-d7cb-4730-9906-7980540813dc"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 146])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 4cb2fda6-91fb-43d0-be7c-77f85347c432, kind: Service { id: 39d14e41-6d5d-474a-aaea-65c36b8f9da5 }, name: Name("nexus-39d14e41-6d5d-474a-aaea-65c36b8f9da5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 243])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7e0b7555-efc0-41d1-b3ba-7ca41e2e30b7, kind: Service { id: 088dbd0f-363c-4785-9d04-aa968256294e }, name: Name("nexus-088dbd0f-363c-4785-9d04-aa968256294e"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 205])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }]} subnet_networks = {Name("external-dns"): [V4(Ipv4Network { addr: 172.30.1.0, prefix: 24 }), V6(Ipv6Network { addr: fd77:e9d2:9cd9:1::, prefix: 64 })], Name("nexus"): [V4(Ipv4Network { addr: 172.30.2.0, prefix: 24 }), V6(Ipv6Network { addr: fd77:e9d2:9cd9:2::, prefix: 64 })]} vpc_interfaces = {}
10122024-11-21T19:33:34.119ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
10132024-11-21T19:33:34.119ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
10142024-11-21T19:33:34.120ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10152024-11-21T19:33:34.122ZDEBGnexus (InternalApi): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Fleet result = Ok(())
10162024-11-21T19:33:34.122ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
10172024-11-21T19:33:34.122ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
10182024-11-21T19:33:34.142ZDEBGnexus (InternalApi): Allowlist for user-facing services is set to allow any inbound traffic. Existing VPC firewall rules will not be modified.
10192024-11-21T19:33:34.153ZDEBGnexus (InternalApi): resolved firewall rules for sled agents sled_agent_rules = [ResolvedVpcFirewallRule { action: Allow, direction: Inbound, filter_hosts: None, filter_ports: Some([L4PortRange("53")]), filter_protocols: Some([Udp]), priority: 65534, status: Enabled, targets: [NetworkInterface { id: 4c211da9-c36c-41bc-8274-9aa6420c20e6, kind: Service { id: 90b200e8-d1ee-4e0d-975c-64628c5532d2 }, name: Name("external-dns-90b200e8-d1ee-4e0d-975c-64628c5532d2"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 185])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b69bf686-c443-40c6-b854-c2ff6e5e107a, kind: Service { id: 4e669ff4-6dde-4cd2-8491-623460b56d8d }, name: Name("external-dns-4e669ff4-6dde-4cd2-8491-623460b56d8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 188])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }] }, ResolvedVpcFirewallRule { action: Allow, direction: Inbound, filter_hosts: None, filter_ports: Some([L4PortRange("80"), L4PortRange("443")]), filter_protocols: Some([Tcp]), priority: 65534, status: Enabled, targets: [NetworkInterface { id: 03ccecdf-44ea-4815-8920-5ebe8d7bf7ff, kind: Service { id: 915e11b8-d7cb-4730-9906-7980540813dc }, name: Name("nexus-915e11b8-d7cb-4730-9906-7980540813dc"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 146])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 4cb2fda6-91fb-43d0-be7c-77f85347c432, kind: Service { id: 39d14e41-6d5d-474a-aaea-65c36b8f9da5 }, name: Name("nexus-39d14e41-6d5d-474a-aaea-65c36b8f9da5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 243])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7e0b7555-efc0-41d1-b3ba-7ca41e2e30b7, kind: Service { id: 088dbd0f-363c-4785-9d04-aa968256294e }, name: Name("nexus-088dbd0f-363c-4785-9d04-aa968256294e"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 205])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }] }]
10202024-11-21T19:33:34.153ZDEBGnexus (InternalApi): resolved 2 rules for sleds
10212024-11-21T19:33:34.259ZDEBGnexus (InternalApi): resolved sleds for vpc oxide-services vpc_to_sled = [Sled { identity: SledIdentity { id: 203795ed-576b-40e4-bed8-7463e3f2162c, time_created: 2024-11-21T19:33:30.261358Z, time_modified: 2024-11-21T19:33:30.261358Z }, time_deleted: None, rcgen: Generation(Generation(19)), rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, is_scrimlet: true, serial_number: "buskin", part_number: "i86pc", revision: SqlU32(0), usable_hardware_threads: SqlU32(48), usable_physical_ram: ByteCount(ByteCount(102952026112)), reservoir_size: ByteCount(ByteCount(51474595840)), ip: fd00:1122:3344:101::1, port: SqlU16(12345), last_used_address: fd00:1122:3344:101::ffff, policy: InService, state: Active, sled_agent_gen: Generation(Generation(1)) }]
10222024-11-21T19:33:34.259ZDEBGnexus (InternalApi): sending firewall rules to sled agents
10232024-11-21T19:33:34.259ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
10242024-11-21T19:33:34.259ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
10252024-11-21T19:33:34.262ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {} }
10262024-11-21T19:33:34.262ZDEBGnexus (InternalApi): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Database result = Ok(())
10272024-11-21T19:33:34.264ZDEBGnexus (InternalApi): roles roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10282024-11-21T19:33:34.266ZDEBGnexus (InternalApi): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
10292024-11-21T19:33:34.299ZDEBGnexus (InternalApi): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc/001de000-074c-4000-8000-000000000000/firewall/rules
10302024-11-21T19:33:34.299ZDEBGnexus (InternalApi): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc/001de000-074c-4000-8000-000000000000/firewall/rules", status: 204, headers: {"x-request-id": "60189648-e862-4d87-8536-fe9122d2b892", "date": "Thu, 21 Nov 2024 19:33:33 GMT"} })
10312024-11-21T19:33:34.300ZINFOnexus (InternalApi): updated firewall rules on 1 sleds file = nexus/networking/src/firewall_rules.rs:496
10322024-11-21T19:33:34.300ZINFOnexus: plumbed initial IP allowlist file = nexus/src/app/allow_list.rs:161
10332024-11-21T19:33:37.827ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 3 reason = Timeout
10342024-11-21T19:33:37.827ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
10352024-11-21T19:33:37.828ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
10362024-11-21T19:33:37.828ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 3 reason = Timeout
10372024-11-21T19:33:37.828ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
10382024-11-21T19:33:37.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
10392024-11-21T19:33:37.829ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10402024-11-21T19:33:37.832ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
10412024-11-21T19:33:37.832ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
10422024-11-21T19:33:37.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
10432024-11-21T19:33:37.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
10442024-11-21T19:33:37.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
10452024-11-21T19:33:37.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10462024-11-21T19:33:37.839ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
10472024-11-21T19:33:37.839ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
10482024-11-21T19:33:37.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
10492024-11-21T19:33:37.856ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
10502024-11-21T19:33:37.856ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
10512024-11-21T19:33:37.857ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 29.18107ms iteration = 3 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
10522024-11-21T19:33:37.871ZINFOnexus (BackgroundTasks): found new target blueprint (first find) background_task = blueprint_loader file = nexus/src/app/background/tasks/blueprint_load.rs:88 target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
10532024-11-21T19:33:37.871ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 2 reason = Dependency
10542024-11-21T19:33:37.871ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 43.486321ms iteration = 3 status = {"enabled":false,"status":"first target blueprint","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z","time_found":"2024-11-21T19:33:37.871166173Z"}
10552024-11-21T19:33:37.871ZDEBGnexus (BackgroundTasks): activating background_task = crdb_node_id_collector iteration = 2 reason = Dependency
10562024-11-21T19:33:37.873ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:83 reason = blueprint disabled target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb
10572024-11-21T19:33:37.873ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 1.976657ms iteration = 2 status = {"enabled":false,"target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb"}
10582024-11-21T19:33:37.873ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
10592024-11-21T19:33:37.873ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
10602024-11-21T19:33:37.873ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
10612024-11-21T19:33:37.873ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
10622024-11-21T19:33:37.873ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
10632024-11-21T19:33:37.873ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
10642024-11-21T19:33:37.873ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
10652024-11-21T19:33:37.873ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
10662024-11-21T19:33:37.873ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
10672024-11-21T19:33:37.874ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
10682024-11-21T19:33:37.875ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10692024-11-21T19:33:37.877ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
10702024-11-21T19:33:37.877ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
10712024-11-21T19:33:37.877ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
10722024-11-21T19:33:37.877ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10732024-11-21T19:33:37.879ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
10742024-11-21T19:33:37.879ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
10752024-11-21T19:33:37.879ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
10762024-11-21T19:33:37.879ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10772024-11-21T19:33:37.881ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
10782024-11-21T19:33:37.881ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
10792024-11-21T19:33:37.881ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
10802024-11-21T19:33:37.881ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10812024-11-21T19:33:37.883ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
10822024-11-21T19:33:37.883ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
10832024-11-21T19:33:37.883ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
10842024-11-21T19:33:37.883ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10852024-11-21T19:33:37.885ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
10862024-11-21T19:33:37.885ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
10872024-11-21T19:33:37.885ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
10882024-11-21T19:33:37.917ZDEBGnexus (BackgroundTasks): client request background_task = crdb_node_id_collector body = None method = GET uri = http://[fd00:1122:3344:101::5]:32222/node/id
10892024-11-21T19:33:37.950ZDEBGnexus (BackgroundTasks): client request background_task = crdb_node_id_collector body = None method = GET uri = http://[fd00:1122:3344:101::7]:32222/node/id
10902024-11-21T19:33:37.983ZDEBGnexus (BackgroundTasks): client request background_task = crdb_node_id_collector body = None method = GET uri = http://[fd00:1122:3344:101::4]:32222/node/id
10912024-11-21T19:33:38.017ZDEBGnexus (BackgroundTasks): client request background_task = crdb_node_id_collector body = None method = GET uri = http://[fd00:1122:3344:101::6]:32222/node/id
10922024-11-21T19:33:38.049ZDEBGnexus (BackgroundTasks): client request background_task = crdb_node_id_collector body = None method = GET uri = http://[fd00:1122:3344:101::3]:32222/node/id
10932024-11-21T19:33:39.826ZDEBGnexus (BackgroundTasks): client response background_task = crdb_node_id_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::6]:32222/node/id", status: 200, headers: {"content-type": "application/json", "x-request-id": "52df2756-f531-4309-a523-e9183caaae1d", "content-length": "64", "date": "Thu, 21 Nov 2024 19:33:39 GMT"} })
10942024-11-21T19:33:39.828ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
10952024-11-21T19:33:39.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
10962024-11-21T19:33:39.830ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
10972024-11-21T19:33:39.832ZDEBGnexus (BackgroundTasks): authorize result action = Modify actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
10982024-11-21T19:33:39.832ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
10992024-11-21T19:33:39.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
11002024-11-21T19:33:40.269ZDEBGnexus (BackgroundTasks): client response background_task = crdb_node_id_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::7]:32222/node/id", status: 200, headers: {"content-type": "application/json", "x-request-id": "94241aa1-417a-4c49-87b4-d2e566a38852", "content-length": "64", "date": "Thu, 21 Nov 2024 19:33:40 GMT"} })
11012024-11-21T19:33:40.269ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
11022024-11-21T19:33:40.270ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
11032024-11-21T19:33:40.271ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11042024-11-21T19:33:40.273ZDEBGnexus (BackgroundTasks): authorize result action = Modify actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
11052024-11-21T19:33:40.273ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
11062024-11-21T19:33:40.273ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
11072024-11-21T19:33:40.443ZDEBGnexus (BackgroundTasks): client response background_task = crdb_node_id_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::3]:32222/node/id", status: 200, headers: {"content-type": "application/json", "x-request-id": "1b7018a2-1db1-45f4-b72f-920951b36a5a", "content-length": "64", "date": "Thu, 21 Nov 2024 19:33:40 GMT"} })
11082024-11-21T19:33:40.443ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
11092024-11-21T19:33:40.443ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
11102024-11-21T19:33:40.444ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11112024-11-21T19:33:40.446ZDEBGnexus (BackgroundTasks): authorize result action = Modify actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
11122024-11-21T19:33:40.446ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
11132024-11-21T19:33:40.446ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
11142024-11-21T19:33:41.481ZDEBGnexus (BackgroundTasks): client response background_task = crdb_node_id_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::5]:32222/node/id", status: 200, headers: {"content-type": "application/json", "x-request-id": "3c84a147-7816-45e0-911c-89afd98da637", "content-length": "64", "date": "Thu, 21 Nov 2024 19:33:41 GMT"} })
11152024-11-21T19:33:41.481ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
11162024-11-21T19:33:41.481ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
11172024-11-21T19:33:41.483ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11182024-11-21T19:33:41.484ZDEBGnexus (BackgroundTasks): authorize result action = Modify actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
11192024-11-21T19:33:41.484ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
11202024-11-21T19:33:41.484ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
11212024-11-21T19:33:47.828ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 2 reason = Timeout
11222024-11-21T19:33:47.828ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
11232024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 4 reason = Timeout
11242024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
11252024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 2 reason = Timeout
11262024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
11272024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 2 reason = Timeout
11282024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 3 reason = Timeout
11292024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
11302024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
11312024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 2 reason = Timeout
11322024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
11332024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 2 reason = Timeout
11342024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
11352024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 2 reason = Timeout
11362024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 2 reason = Timeout
11372024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
11382024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 2 reason = Timeout
11392024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
11402024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 4 reason = Timeout
11412024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
11422024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 2 reason = Timeout
11432024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
11442024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 2 reason = Timeout
11452024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
11462024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
11472024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 2 reason = Timeout
11482024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
11492024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 2 reason = Timeout
11502024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
11512024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 2 reason = Timeout
11522024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
11532024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 2 reason = Timeout
11542024-11-21T19:33:47.829ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
11552024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
11562024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
11572024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
11582024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
11592024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
11602024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
11612024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
11622024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
11632024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
11642024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
11652024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
11662024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
11672024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
11682024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
11692024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
11702024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
11712024-11-21T19:33:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
11722024-11-21T19:33:47.832ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 770, tv_nsec: 713089991 } })
11732024-11-21T19:33:47.832ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11742024-11-21T19:33:47.832ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
11752024-11-21T19:33:47.833ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11762024-11-21T19:33:47.834ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 5.234ms iteration = 2 status = {"errors":[],"records_set_to_done":[]}
11772024-11-21T19:33:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
11782024-11-21T19:33:47.835ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
11792024-11-21T19:33:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
11802024-11-21T19:33:47.836ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11812024-11-21T19:33:47.836ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 7.753913ms iteration = 2 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
11822024-11-21T19:33:47.836ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 770, tv_nsec: 717129075 } })
11832024-11-21T19:33:47.836ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
11842024-11-21T19:33:47.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
11852024-11-21T19:33:47.836ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
11862024-11-21T19:33:47.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
11872024-11-21T19:33:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
11882024-11-21T19:33:47.838ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
11892024-11-21T19:33:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
11902024-11-21T19:33:47.840ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11912024-11-21T19:33:47.842ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11922024-11-21T19:33:47.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
11932024-11-21T19:33:47.842ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
11942024-11-21T19:33:47.843ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
11952024-11-21T19:33:47.844ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
11962024-11-21T19:33:47.845ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
11972024-11-21T19:33:47.845ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
11982024-11-21T19:33:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
11992024-11-21T19:33:47.847ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
12002024-11-21T19:33:47.847ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
12012024-11-21T19:33:47.847ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
12022024-11-21T19:33:47.850ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
12032024-11-21T19:33:47.850ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
12042024-11-21T19:33:47.852ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12052024-11-21T19:33:47.854ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
12062024-11-21T19:33:47.854ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12072024-11-21T19:33:47.854ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
12082024-11-21T19:33:47.854ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
12092024-11-21T19:33:47.854ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
12102024-11-21T19:33:47.855ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
12112024-11-21T19:33:47.855ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
12122024-11-21T19:33:47.855ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
12132024-11-21T19:33:47.858ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 770, tv_nsec: 738919046 } })
12142024-11-21T19:33:47.858ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
12152024-11-21T19:33:47.859ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
12162024-11-21T19:33:47.859ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
12172024-11-21T19:33:47.859ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
12182024-11-21T19:33:47.863ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
12192024-11-21T19:33:47.864ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
12202024-11-21T19:33:47.887ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
12212024-11-21T19:33:47.887ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 59.165821ms iteration = 4 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
12222024-11-21T19:33:47.889ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
12232024-11-21T19:33:47.890ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
12242024-11-21T19:33:47.891ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
12252024-11-21T19:33:47.891ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 62.035809ms iteration = 2 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
12262024-11-21T19:33:47.891ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
12272024-11-21T19:33:47.891ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
12282024-11-21T19:33:47.893ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
12292024-11-21T19:33:47.893ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
12302024-11-21T19:33:47.893ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
12312024-11-21T19:33:47.893ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 64.579383ms iteration = 4 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
12322024-11-21T19:33:47.893ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
12332024-11-21T19:33:47.895ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
12342024-11-21T19:33:47.896ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
12352024-11-21T19:33:47.899ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 70.837526ms iteration = 2 status = {"errors":[],"garbage_collect_requested":[]}
12362024-11-21T19:33:47.900ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
12372024-11-21T19:33:47.900ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
12382024-11-21T19:33:47.900ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "827a2f24-f8d5-4c66-9002-581bdb0c7430", "content-length": "26", "date": "Thu, 21 Nov 2024 19:33:47 GMT"} })
12392024-11-21T19:33:47.900ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
12402024-11-21T19:33:47.901ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
12412024-11-21T19:33:47.904ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12422024-11-21T19:33:47.904ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
12432024-11-21T19:33:47.905ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
12442024-11-21T19:33:47.906ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
12452024-11-21T19:33:47.906ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
12462024-11-21T19:33:47.906ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
12472024-11-21T19:33:47.906ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
12482024-11-21T19:33:47.906ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "c5a551e7-4c96-40c2-9b9c-8d6cc809f108", "content-length": "26", "date": "Thu, 21 Nov 2024 19:33:47 GMT"} })
12492024-11-21T19:33:47.906ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
12502024-11-21T19:33:47.906ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
12512024-11-21T19:33:47.907ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 78.835504ms iteration = 2 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
12522024-11-21T19:33:47.907ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
12532024-11-21T19:33:47.918ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
12542024-11-21T19:33:47.918ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
12552024-11-21T19:33:47.922ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
12562024-11-21T19:33:47.923ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
12572024-11-21T19:33:47.924ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12582024-11-21T19:33:47.926ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 97.000476ms iteration = 2 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
12592024-11-21T19:33:47.927ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
12602024-11-21T19:33:47.927ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
12612024-11-21T19:33:47.927ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
12622024-11-21T19:33:47.933ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
12632024-11-21T19:33:47.933ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
12642024-11-21T19:33:47.934ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
12652024-11-21T19:33:47.934ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
12662024-11-21T19:33:47.939ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
12672024-11-21T19:33:47.939ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
12682024-11-21T19:33:47.939ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
12692024-11-21T19:33:47.939ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
12702024-11-21T19:33:47.952ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 123.298558ms iteration = 3 status = {"physical_disks_added":0}
12712024-11-21T19:33:47.964ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
12722024-11-21T19:33:47.964ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
12732024-11-21T19:33:47.965ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "15696c69-bbe9-4b92-ad68-d58e9922cd2b", "content-length": "26", "date": "Thu, 21 Nov 2024 19:33:47 GMT"} })
12742024-11-21T19:33:47.965ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
12752024-11-21T19:33:47.966ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
12762024-11-21T19:33:47.971ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
12772024-11-21T19:33:47.972ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
12782024-11-21T19:33:48.013ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 185.051652ms iteration = 2 status = {}
12792024-11-21T19:33:48.018ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
12802024-11-21T19:33:48.021ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "0890ab5d-1818-4283-800e-9af752f739d6", "content-length": "1", "date": "Thu, 21 Nov 2024 19:33:47 GMT"} })
12812024-11-21T19:33:48.022ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
12822024-11-21T19:33:48.023ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
12832024-11-21T19:33:48.035ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
12842024-11-21T19:33:48.035ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
12852024-11-21T19:33:48.040ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12862024-11-21T19:33:48.041ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
12872024-11-21T19:33:48.041ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
12882024-11-21T19:33:48.042ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
12892024-11-21T19:33:48.042ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
12902024-11-21T19:33:48.043ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
12912024-11-21T19:33:48.044ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 215.5419ms iteration = 2 status = null
12922024-11-21T19:33:48.045ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
12932024-11-21T19:33:48.046ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 770, tv_nsec: 926637405 } })
12942024-11-21T19:33:48.046ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = service_zone_nat_tracker file = nexus/src/app/mod.rs:1131
12952024-11-21T19:33:48.047ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
12962024-11-21T19:33:48.047ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
12972024-11-21T19:33:48.047ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
12982024-11-21T19:33:48.074ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
12992024-11-21T19:33:48.074ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
13002024-11-21T19:33:48.074ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 244.940873ms iteration = 2 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
13012024-11-21T19:33:48.092ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 263.398899ms iteration = 2 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
13022024-11-21T19:33:48.110ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = service_zone_nat_tracker file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
13032024-11-21T19:33:48.110ZDEBGnexus (MgsClient): client request background_task = service_zone_nat_tracker body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
13042024-11-21T19:33:48.110ZDEBGnexus (MgsClient): client response background_task = service_zone_nat_tracker result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "c6cefc02-5bf0-4e16-8ab3-af5bb689c22c", "content-length": "26", "date": "Thu, 21 Nov 2024 19:33:47 GMT"} })
13052024-11-21T19:33:48.110ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = service_zone_nat_tracker file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
13062024-11-21T19:33:48.112ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = service_zone_nat_tracker file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
13072024-11-21T19:33:48.202ZDEBGnexus (DpdClient): client request background_task = service_zone_nat_tracker body = None method = POST uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/trigger
13082024-11-21T19:33:48.202ZDEBGnexus (DpdClient): client response background_task = service_zone_nat_tracker result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/trigger", status: 200, headers: {"content-type": "application/json", "x-request-id": "1796e641-a26f-4939-8adf-cba7b99bb062", "content-length": "4", "date": "Thu, 21 Nov 2024 19:33:48 GMT"} })
13092024-11-21T19:33:48.203ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 375.018051ms iteration = 2 status = 1
13102024-11-21T19:33:48.207ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
13112024-11-21T19:33:48.208ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
13122024-11-21T19:33:48.208ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "335c915e-a956-4f60-9c55-76c4b4998e2d", "content-length": "691", "date": "Thu, 21 Nov 2024 19:33:47 GMT"} })
13132024-11-21T19:33:48.208ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
13142024-11-21T19:33:48.211ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
13152024-11-21T19:33:48.211ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
13162024-11-21T19:33:48.211ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
13172024-11-21T19:33:48.212ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 383.97653ms iteration = 2 status = {}
13182024-11-21T19:33:48.214ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
13192024-11-21T19:33:48.214ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
13202024-11-21T19:33:48.214ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
13212024-11-21T19:33:48.223ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "89a2667d-defb-4fdf-8cb3-810efe5b3e4a", "content-length": "372", "date": "Thu, 21 Nov 2024 19:33:48 GMT"} })
13222024-11-21T19:33:48.230ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
13232024-11-21T19:33:48.230ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
13242024-11-21T19:33:48.237ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
13252024-11-21T19:33:48.237ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "1fabb970-2973-4069-bd32-72935b2b1be1", "content-length": "134", "date": "Thu, 21 Nov 2024 19:33:48 GMT"} })
13262024-11-21T19:33:48.238ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
13272024-11-21T19:33:48.238ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
13282024-11-21T19:33:48.241ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
13292024-11-21T19:33:48.241ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
13302024-11-21T19:33:48.241ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
13312024-11-21T19:33:48.241ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
13322024-11-21T19:33:48.241ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
13332024-11-21T19:33:48.241ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
13342024-11-21T19:33:48.243ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
13352024-11-21T19:33:48.243ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
13362024-11-21T19:33:48.243ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
13372024-11-21T19:33:48.243ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
13382024-11-21T19:33:48.244ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
13392024-11-21T19:33:48.250ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
13402024-11-21T19:33:48.250ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "e4dd89eb-cd44-4fc0-9743-68d4409e8200", "content-length": "2", "date": "Thu, 21 Nov 2024 19:33:48 GMT"} })
13412024-11-21T19:33:48.250ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
13422024-11-21T19:33:48.250ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "68d361a5-287f-4599-82df-178f66875b74", "content-length": "51", "date": "Thu, 21 Nov 2024 19:33:48 GMT"} })
13432024-11-21T19:33:48.250ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
13442024-11-21T19:33:48.253ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
13452024-11-21T19:33:48.282ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
13462024-11-21T19:33:48.283ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
13472024-11-21T19:33:48.283ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "82631978-904a-4f35-8f69-815ffc941343", "date": "Thu, 21 Nov 2024 19:33:47 GMT"} })
13482024-11-21T19:33:48.283ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
13492024-11-21T19:33:48.283ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
13502024-11-21T19:33:48.288ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
13512024-11-21T19:33:48.288ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
13522024-11-21T19:33:48.292ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
13532024-11-21T19:33:48.292ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
13542024-11-21T19:33:48.296ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
13552024-11-21T19:33:48.296ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
13562024-11-21T19:33:48.302ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "6aeaf807-467c-4aad-b50c-f9ab13b5ca28", "date": "Thu, 21 Nov 2024 19:33:48 GMT"} })
13572024-11-21T19:33:48.303ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 474.383678ms iteration = 2 status = {}
13582024-11-21T19:33:48.454ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "bcd171f6-76c0-4b64-9e78-5dae388eacd0", "date": "Thu, 21 Nov 2024 19:33:48 GMT"} })
13592024-11-21T19:33:48.454ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
13602024-11-21T19:33:48.455ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
13612024-11-21T19:33:48.455ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "5fdc030f-2d39-4799-8f47-c3016e6ac448", "content-length": "682", "date": "Thu, 21 Nov 2024 19:33:48 GMT"} })
13622024-11-21T19:33:48.455ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
13632024-11-21T19:33:48.455ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
13642024-11-21T19:33:48.473ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
13652024-11-21T19:33:48.473ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
13662024-11-21T19:33:48.477ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
13672024-11-21T19:33:48.478ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
13682024-11-21T19:33:48.480ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
13692024-11-21T19:33:48.480ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
13702024-11-21T19:33:48.508ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 679.706422ms iteration = 2 status = {}
13712024-11-21T19:33:52.984ZDEBGnexus (BackgroundTasks): client response background_task = crdb_node_id_collector result = Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::4]:32222/node/id", source: TimedOut })
13722024-11-21T19:33:52.987ZDEBGnexus (BackgroundTasks): activation complete background_task = crdb_node_id_collector elapsed = 15.116094986s iteration = 2 status = {"errors":[{"err":"failed to fetch node ID for zone 79d9d42d-4227-48a4-8c55-f6743d9bd839 at http://[fd00:1122:3344:101::4]:32222: Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:32222/node/id): error sending request for url (http://[fd00:1122:3344:101::4]:32222/node/id): operation timed out","zone_id":"79d9d42d-4227-48a4-8c55-f6743d9bd839"}],"nsuccess":4}
13732024-11-21T19:33:57.828ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 5 reason = Timeout
13742024-11-21T19:33:57.828ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
13752024-11-21T19:33:57.828ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
13762024-11-21T19:33:57.829ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 5 reason = Timeout
13772024-11-21T19:33:57.829ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
13782024-11-21T19:33:57.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
13792024-11-21T19:33:57.830ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
13802024-11-21T19:33:57.832ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
13812024-11-21T19:33:57.832ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
13822024-11-21T19:33:57.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
13832024-11-21T19:33:57.835ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
13842024-11-21T19:33:57.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
13852024-11-21T19:33:57.858ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
13862024-11-21T19:33:57.858ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
13872024-11-21T19:33:57.888ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 58.990868ms iteration = 5 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
13882024-11-21T19:33:57.932ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
13892024-11-21T19:33:57.935ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
13902024-11-21T19:33:57.935ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
13912024-11-21T19:33:57.935ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
13922024-11-21T19:33:57.947ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
13932024-11-21T19:33:57.947ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 119.660409ms iteration = 5 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
13942024-11-21T19:34:07.828ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 6 reason = Timeout
13952024-11-21T19:34:07.828ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
13962024-11-21T19:34:07.828ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
13972024-11-21T19:34:07.829ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 6 reason = Timeout
13982024-11-21T19:34:07.829ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
13992024-11-21T19:34:07.830ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
14002024-11-21T19:34:07.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
14012024-11-21T19:34:07.832ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
14022024-11-21T19:34:07.832ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
14032024-11-21T19:34:07.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
14042024-11-21T19:34:07.835ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
14052024-11-21T19:34:07.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
14062024-11-21T19:34:07.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
14072024-11-21T19:34:07.839ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
14082024-11-21T19:34:07.839ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
14092024-11-21T19:34:07.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
14102024-11-21T19:34:07.855ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
14112024-11-21T19:34:07.855ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 27.299025ms iteration = 6 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
14122024-11-21T19:34:07.858ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
14132024-11-21T19:34:07.858ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
14142024-11-21T19:34:07.859ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 29.586636ms iteration = 6 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
14152024-11-21T19:34:17.828ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 3 reason = Timeout
14162024-11-21T19:34:17.828ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
14172024-11-21T19:34:17.828ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 7 reason = Timeout
14182024-11-21T19:34:17.828ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 3 reason = Timeout
14192024-11-21T19:34:17.828ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
14202024-11-21T19:34:17.828ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:83 reason = blueprint disabled target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb
14212024-11-21T19:34:17.828ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 17.33\u{b5}s iteration = 3 status = {"enabled":false,"target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb"}
14222024-11-21T19:34:17.828ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 3 reason = Timeout
14232024-11-21T19:34:17.828ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 3 reason = Timeout
14242024-11-21T19:34:17.828ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
14252024-11-21T19:34:17.828ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
14262024-11-21T19:34:17.828ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 3 reason = Timeout
14272024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
14282024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 2 reason = Timeout
14292024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 2 reason = Timeout
14302024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 3 reason = Timeout
14312024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
14322024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
14332024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 2 reason = Timeout
14342024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
14352024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
14362024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 2 reason = Timeout
14372024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
14382024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
14392024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 2 reason = Timeout
14402024-11-21T19:34:17.829ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
14412024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
14422024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 3 reason = Timeout
14432024-11-21T19:34:17.829ZWARNnexus (BackgroundTasks): DNS propagation: skipped background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:69 reason = no config
14442024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 2 reason = Timeout
14452024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 10.2\u{b5}s iteration = 3 status = {"error":"no config"}
14462024-11-21T19:34:17.829ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-11-21 19:24:17.829019192 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
14472024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 3 reason = Timeout
14482024-11-21T19:34:17.829ZWARNnexus (BackgroundTasks): DNS propagation: skipped background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:69 reason = no config
14492024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
14502024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 2 reason = Timeout
14512024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 10.121\u{b5}s iteration = 3 status = {"error":"no config"}
14522024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 3 reason = Timeout
14532024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 4 reason = Timeout
14542024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 3 reason = Timeout
14552024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
14562024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
14572024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = instance_reincarnation iteration = 2 reason = Timeout
14582024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 3 reason = Timeout
14592024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 7 reason = Timeout
14602024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
14612024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
14622024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
14632024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 2 reason = Timeout
14642024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
14652024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 3 reason = Timeout
14662024-11-21T19:34:17.829ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
14672024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
14682024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 3 reason = Timeout
14692024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
14702024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 3 reason = Timeout
14712024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
14722024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
14732024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 3 reason = Timeout
14742024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
14752024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 3 reason = Timeout
14762024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
14772024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 3 reason = Timeout
14782024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
14792024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 2 reason = Timeout
14802024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
14812024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
14822024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
14832024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
14842024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
14852024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
14862024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
14872024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
14882024-11-21T19:34:17.829ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("4e669ff4-6dde-4cd2-8491-623460b56d8d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 800, tv_nsec: 709594155 } })
14892024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
14902024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
14912024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
14922024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
14932024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
14942024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
14952024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
14962024-11-21T19:34:17.829ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0377cf96-baf3-4e09-a570-eb7c67037d4d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0d1b3b31-4591-4bcb-931d-e8e03365f0d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:1::1))) }], valid_until: Instant { tv_sec: 800, tv_nsec: 709796027 } })
14972024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
14982024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
14992024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
15002024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
15012024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
15022024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
15032024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
15042024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
15052024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
15062024-11-21T19:34:17.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
15072024-11-21T19:34:17.830ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
15082024-11-21T19:34:17.830ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 1.247276ms iteration = 2 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
15092024-11-21T19:34:17.830ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } dns_group = internal
15102024-11-21T19:34:17.830ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 1.466559ms iteration = 2 status = {"addresses":["[fd00:1122:3344:3::1]:5353","[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353"]}
15112024-11-21T19:34:17.832ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 800, tv_nsec: 712417392 } })
15122024-11-21T19:34:17.832ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
15132024-11-21T19:34:17.832ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.76564ms iteration = 3 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
15142024-11-21T19:34:17.834ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15152024-11-21T19:34:17.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15162024-11-21T19:34:17.835ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15172024-11-21T19:34:17.837ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
15182024-11-21T19:34:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
15192024-11-21T19:34:17.838ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
15202024-11-21T19:34:17.838ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
15212024-11-21T19:34:17.838ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 800, tv_nsec: 718230620 } })
15222024-11-21T19:34:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
15232024-11-21T19:34:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
15242024-11-21T19:34:17.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
15252024-11-21T19:34:17.838ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
15262024-11-21T19:34:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
15272024-11-21T19:34:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
15282024-11-21T19:34:17.839ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
15292024-11-21T19:34:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
15302024-11-21T19:34:17.842ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-11-21 19:24:17.829019192 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
15312024-11-21T19:34:17.842ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 13.364389ms iteration = 2 status = {"expiration":"2024-11-21T19:24:17.829019192Z","pruned":[]}
15322024-11-21T19:34:17.845ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15332024-11-21T19:34:17.846ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
15342024-11-21T19:34:17.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
15352024-11-21T19:34:17.846ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15362024-11-21T19:34:17.848ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
15372024-11-21T19:34:17.848ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
15382024-11-21T19:34:17.849ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
15392024-11-21T19:34:17.850ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
15402024-11-21T19:34:17.850ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
15412024-11-21T19:34:17.850ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
15422024-11-21T19:34:17.850ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15432024-11-21T19:34:17.853ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
15442024-11-21T19:34:17.853ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
15452024-11-21T19:34:17.853ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
15462024-11-21T19:34:17.859ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 30.278485ms iteration = 2 status = {"errors":[],"found_port_ok":[]}
15472024-11-21T19:34:17.862ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
15482024-11-21T19:34:17.862ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
15492024-11-21T19:34:17.862ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
15502024-11-21T19:34:17.863ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
15512024-11-21T19:34:17.869ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 800, tv_nsec: 749619339 } })
15522024-11-21T19:34:17.870ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
15532024-11-21T19:34:17.871ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
15542024-11-21T19:34:17.871ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
15552024-11-21T19:34:17.876ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
15562024-11-21T19:34:17.877ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
15572024-11-21T19:34:17.902ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15582024-11-21T19:34:17.904ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
15592024-11-21T19:34:17.904ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
15602024-11-21T19:34:17.904ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
15612024-11-21T19:34:17.908ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
15622024-11-21T19:34:17.909ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
15632024-11-21T19:34:17.909ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 80.547666ms iteration = 3 status = {"errors":[],"records_set_to_done":[]}
15642024-11-21T19:34:17.911ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 82.776496ms iteration = 3 status = {"errors":[],"garbage_collect_requested":[]}
15652024-11-21T19:34:17.912ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
15662024-11-21T19:34:17.912ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15672024-11-21T19:34:17.912ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15682024-11-21T19:34:17.912ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
15692024-11-21T19:34:17.913ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
15702024-11-21T19:34:17.913ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
15712024-11-21T19:34:17.913ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
15722024-11-21T19:34:17.913ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
15732024-11-21T19:34:17.915ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
15742024-11-21T19:34:17.915ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
15752024-11-21T19:34:17.915ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
15762024-11-21T19:34:17.915ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
15772024-11-21T19:34:17.915ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
15782024-11-21T19:34:17.917ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
15792024-11-21T19:34:17.917ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
15802024-11-21T19:34:17.917ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
15812024-11-21T19:34:17.920ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15822024-11-21T19:34:17.922ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0 total = 0
15832024-11-21T19:34:17.922ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:171 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0 vmms_found = 0
15842024-11-21T19:34:17.922ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 93.096484ms iteration = 2 status = {"errors":[],"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0,"vmms_found":0}
15852024-11-21T19:34:17.923ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15862024-11-21T19:34:17.923ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
15872024-11-21T19:34:17.923ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
15882024-11-21T19:34:17.923ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
15892024-11-21T19:34:17.924ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
15902024-11-21T19:34:17.924ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
15912024-11-21T19:34:17.926ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
15922024-11-21T19:34:17.929ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
15932024-11-21T19:34:17.929ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
15942024-11-21T19:34:17.929ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
15952024-11-21T19:34:17.930ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
15962024-11-21T19:34:17.930ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
15972024-11-21T19:34:17.931ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
15982024-11-21T19:34:17.931ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
15992024-11-21T19:34:17.935ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
16002024-11-21T19:34:17.935ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
16012024-11-21T19:34:17.935ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
16022024-11-21T19:34:17.937ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16032024-11-21T19:34:17.939ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16042024-11-21T19:34:17.941ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
16052024-11-21T19:34:17.941ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
16062024-11-21T19:34:17.941ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
16072024-11-21T19:34:17.942ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
16082024-11-21T19:34:17.942ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
16092024-11-21T19:34:17.942ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
16102024-11-21T19:34:17.943ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16112024-11-21T19:34:17.945ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
16122024-11-21T19:34:17.945ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
16132024-11-21T19:34:17.945ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
16142024-11-21T19:34:17.949ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
16152024-11-21T19:34:17.949ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
16162024-11-21T19:34:17.951ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
16172024-11-21T19:34:17.951ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
16182024-11-21T19:34:17.954ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
16192024-11-21T19:34:17.954ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 125.471056ms iteration = 3 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
16202024-11-21T19:34:17.955ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
16212024-11-21T19:34:17.956ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
16222024-11-21T19:34:17.967ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
16232024-11-21T19:34:17.967ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 138.741934ms iteration = 7 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
16242024-11-21T19:34:17.972ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
16252024-11-21T19:34:17.972ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
16262024-11-21T19:34:17.973ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 143.927003ms iteration = 7 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
16272024-11-21T19:34:17.975ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
16282024-11-21T19:34:17.975ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
16292024-11-21T19:34:17.975ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 146.133422ms iteration = 2 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
16302024-11-21T19:34:17.975ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 146.155623ms iteration = 3 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
16312024-11-21T19:34:17.975ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
16322024-11-21T19:34:17.975ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
16332024-11-21T19:34:17.975ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
16342024-11-21T19:34:17.975ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
16352024-11-21T19:34:17.975ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
16362024-11-21T19:34:17.976ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 147.239598ms iteration = 3 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
16372024-11-21T19:34:17.977ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16382024-11-21T19:34:17.981ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
16392024-11-21T19:34:17.981ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
16402024-11-21T19:34:17.981ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
16412024-11-21T19:34:17.981ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
16422024-11-21T19:34:17.981ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
16432024-11-21T19:34:17.981ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
16442024-11-21T19:34:17.981ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
16452024-11-21T19:34:17.981ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
16462024-11-21T19:34:17.983ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
16472024-11-21T19:34:17.983ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
16482024-11-21T19:34:17.984ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16492024-11-21T19:34:17.985ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16502024-11-21T19:34:17.986ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
16512024-11-21T19:34:17.987ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
16522024-11-21T19:34:17.987ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
16532024-11-21T19:34:17.988ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "ced9b0e5-fc2a-4274-a571-445221b84334", "content-length": "26", "date": "Thu, 21 Nov 2024 19:34:17 GMT"} })
16542024-11-21T19:34:17.988ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
16552024-11-21T19:34:17.989ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
16562024-11-21T19:34:17.989ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
16572024-11-21T19:34:17.989ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
16582024-11-21T19:34:17.989ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
16592024-11-21T19:34:17.990ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
16602024-11-21T19:34:17.990ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
16612024-11-21T19:34:17.991ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
16622024-11-21T19:34:17.991ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
16632024-11-21T19:34:17.991ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 162.075596ms iteration = 3 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
16642024-11-21T19:34:17.995ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16652024-11-21T19:34:17.998ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
16662024-11-21T19:34:17.998ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
16672024-11-21T19:34:17.998ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
16682024-11-21T19:34:17.999ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16692024-11-21T19:34:18.000ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 171.872187ms iteration = 3 status = 0
16702024-11-21T19:34:18.001ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
16712024-11-21T19:34:18.001ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
16722024-11-21T19:34:18.001ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
16732024-11-21T19:34:18.003ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
16742024-11-21T19:34:18.003ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
16752024-11-21T19:34:18.003ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "dbd48832-6a95-4f9b-9670-db074b165bb7", "content-length": "26", "date": "Thu, 21 Nov 2024 19:34:17 GMT"} })
16762024-11-21T19:34:18.003ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
16772024-11-21T19:34:18.004ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
16782024-11-21T19:34:18.013ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
16792024-11-21T19:34:18.013ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
16802024-11-21T19:34:18.014ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "82bfbe8e-9783-4298-be25-5984278d2deb", "content-length": "26", "date": "Thu, 21 Nov 2024 19:34:17 GMT"} })
16812024-11-21T19:34:18.014ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
16822024-11-21T19:34:18.015ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 185.985435ms iteration = 3 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
16832024-11-21T19:34:18.015ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
16842024-11-21T19:34:18.021ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
16852024-11-21T19:34:18.021ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
16862024-11-21T19:34:18.021ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
16872024-11-21T19:34:18.022ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
16882024-11-21T19:34:18.027ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16892024-11-21T19:34:18.027ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
16902024-11-21T19:34:18.027ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 0 version = 2
16912024-11-21T19:34:18.027ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
16922024-11-21T19:34:18.027ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
16932024-11-21T19:34:18.028ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
16942024-11-21T19:34:18.032ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
16952024-11-21T19:34:18.032ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
16962024-11-21T19:34:18.032ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
16972024-11-21T19:34:18.033ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
16982024-11-21T19:34:18.033ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
16992024-11-21T19:34:18.033ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
17002024-11-21T19:34:18.037ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 1 version = 2
17012024-11-21T19:34:18.037ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
17022024-11-21T19:34:18.037ZINFOnexus (BackgroundTasks): found latest generation (first find) background_task = dns_config_external dns_group = external file = nexus/src/app/background/tasks/dns_config.rs:87 generation = 2
17032024-11-21T19:34:18.037ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 208.088031ms iteration = 2 status = {"generation":2}
17042024-11-21T19:34:18.037ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 4 reason = Dependency
17052024-11-21T19:34:18.040ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 51 version = 1
17062024-11-21T19:34:18.040ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
17072024-11-21T19:34:18.040ZINFOnexus (BackgroundTasks): found latest generation (first find) background_task = dns_config_internal dns_group = internal file = nexus/src/app/background/tasks/dns_config.rs:87 generation = 1
17082024-11-21T19:34:18.040ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 211.098331ms iteration = 2 status = {"generation":1}
17092024-11-21T19:34:18.040ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 4 reason = Dependency
17102024-11-21T19:34:18.055ZINFOnexus (BackgroundTasks): found Silo/DNS/TLS config (initial) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-11-21T19:33:15.446964Z, time_modified: 2024-11-21T19:33:15.446964Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(2)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(2)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }) } file = nexus/src/app/background/tasks/external_endpoints.rs:77
17112024-11-21T19:34:18.056ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 227.065214ms iteration = 2 status = {"by_dns_name":{"default-silo.sys.oxide.test":{"silo_id":"001de000-5110-4000-8000-000000000000","tls_certs":[]},"recovery.sys.oxide.test":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]}},"default_endpoint":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]},"warnings":["silo 001de000-5110-4000-8000-000000000000 with DNS name \\"default-silo.sys.oxide.test\\" has no usable certificates"]}
17122024-11-21T19:34:18.057ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 228.845558ms iteration = 4 status = {"physical_disks_added":0}
17132024-11-21T19:34:18.070ZWARNnexus (BackgroundTasks): instance reincarnation completed with errors background_task = instance_reincarnation file = nexus/src/app/background/tasks/instance_reincarnation.rs:103 instances_changed_state = 0 instances_found = 0 instances_reincarnated = 0 query_errors = 0 restart_errors = 0
17142024-11-21T19:34:18.070ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_reincarnation elapsed = 241.69174ms iteration = 2 status = {"changed_state":[],"disabled":false,"errors":[],"instances_found":{"Failed":0,"SagaUnwound":0},"instances_reincarnated":[],"restart_errors":[]}
17152024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = / versions = all
17162024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /access versions = all
17172024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /assets/{path} versions = all
17182024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /device/auth versions = all
17192024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /device/confirm versions = all
17202024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /device/success versions = all
17212024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /device/token versions = all
17222024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /device/verify versions = all
17232024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /experimental/v1/probes versions = all
17242024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /experimental/v1/probes versions = all
17252024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /experimental/v1/probes/{probe} versions = all
17262024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /experimental/v1/probes/{probe} versions = all
17272024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /images versions = all
17282024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /login versions = all
17292024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /login/{silo_name}/local versions = all
17302024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /login/{silo_name}/saml/{provider_name} versions = all
17312024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /login/{silo_name}/saml/{provider_name} versions = all
17322024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /login/{silo_name}/saml/{provider_name}/redirect versions = all
17332024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /lookup/{path} versions = all
17342024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /projects/{path} versions = all
17352024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /projects-new versions = all
17362024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /settings/{path} versions = all
17372024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /system/{path} versions = all
17382024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /utilization versions = all
17392024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/certificates versions = all
17402024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/certificates versions = all
17412024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/certificates/{certificate} versions = all
17422024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/certificates/{certificate} versions = all
17432024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/disks versions = all
17442024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/disks versions = all
17452024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/disks/{disk} versions = all
17462024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/disks/{disk} versions = all
17472024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/disks/{disk}/bulk-write versions = all
17482024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/disks/{disk}/bulk-write-start versions = all
17492024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/disks/{disk}/bulk-write-stop versions = all
17502024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/disks/{disk}/finalize versions = all
17512024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/disks/{disk}/metrics/{metric} versions = all
17522024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/floating-ips versions = all
17532024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/floating-ips versions = all
17542024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/floating-ips/{floating_ip} versions = all
17552024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/floating-ips/{floating_ip} versions = all
17562024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/floating-ips/{floating_ip} versions = all
17572024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/floating-ips/{floating_ip}/attach versions = all
17582024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/floating-ips/{floating_ip}/detach versions = all
17592024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/groups versions = all
17602024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/groups/{group_id} versions = all
17612024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/images versions = all
17622024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/images versions = all
17632024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/images/{image} versions = all
17642024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/images/{image} versions = all
17652024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/images/{image}/demote versions = all
17662024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/images/{image}/promote versions = all
17672024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/instances versions = all
17682024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/instances versions = all
17692024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/instances/{instance} versions = all
17702024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/instances/{instance} versions = all
17712024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/instances/{instance} versions = all
17722024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/instances/{instance}/disks versions = all
17732024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/instances/{instance}/disks/attach versions = all
17742024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/instances/{instance}/disks/detach versions = all
17752024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/instances/{instance}/external-ips versions = all
17762024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/instances/{instance}/external-ips/ephemeral versions = all
17772024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/instances/{instance}/external-ips/ephemeral versions = all
17782024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/instances/{instance}/reboot versions = all
17792024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/instances/{instance}/serial-console versions = all
17802024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/instances/{instance}/serial-console/stream versions = all
17812024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/instances/{instance}/ssh-public-keys versions = all
17822024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/instances/{instance}/start versions = all
17832024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/instances/{instance}/stop versions = all
17842024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/internet-gateway-ip-addresses versions = all
17852024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/internet-gateway-ip-addresses versions = all
17862024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/internet-gateway-ip-addresses/{address} versions = all
17872024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/internet-gateway-ip-pools versions = all
17882024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/internet-gateway-ip-pools versions = all
17892024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/internet-gateway-ip-pools/{pool} versions = all
17902024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/internet-gateways versions = all
17912024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/internet-gateways versions = all
17922024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/internet-gateways/{gateway} versions = all
17932024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/internet-gateways/{gateway} versions = all
17942024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/ip-pools versions = all
17952024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/ip-pools/{pool} versions = all
17962024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/login/{silo_name}/local versions = all
17972024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/logout versions = all
17982024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/me versions = all
17992024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/me/groups versions = all
18002024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/me/ssh-keys versions = all
18012024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/me/ssh-keys versions = all
18022024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/me/ssh-keys/{ssh_key} versions = all
18032024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/me/ssh-keys/{ssh_key} versions = all
18042024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/metrics/{metric_name} versions = all
18052024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/network-interfaces versions = all
18062024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/network-interfaces versions = all
18072024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/network-interfaces/{interface} versions = all
18082024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/network-interfaces/{interface} versions = all
18092024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/network-interfaces/{interface} versions = all
18102024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/ping versions = all
18112024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/policy versions = all
18122024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/policy versions = all
18132024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/projects versions = all
18142024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/projects versions = all
18152024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/projects/{project} versions = all
18162024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/projects/{project} versions = all
18172024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/projects/{project} versions = all
18182024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/projects/{project}/policy versions = all
18192024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/projects/{project}/policy versions = all
18202024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/snapshots versions = all
18212024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/snapshots versions = all
18222024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/snapshots/{snapshot} versions = all
18232024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/snapshots/{snapshot} versions = all
18242024-11-21T19:34:18.108ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/hardware/disks versions = all
18252024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/hardware/disks/{disk_id} versions = all
18262024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/hardware/racks versions = all
18272024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/hardware/racks/{rack_id} versions = all
18282024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/hardware/sleds versions = all
18292024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/hardware/sleds versions = all
18302024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/hardware/sleds/{sled_id} versions = all
18312024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/hardware/sleds/{sled_id}/disks versions = all
18322024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/hardware/sleds/{sled_id}/instances versions = all
18332024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/system/hardware/sleds/{sled_id}/provision-policy versions = all
18342024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/hardware/sleds-uninitialized versions = all
18352024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/hardware/switch-port versions = all
18362024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/system/hardware/switch-port/{port}/settings versions = all
18372024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/hardware/switch-port/{port}/settings versions = all
18382024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/hardware/switch-port/{port}/status versions = all
18392024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/hardware/switches versions = all
18402024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/hardware/switches/{switch_id} versions = all
18412024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/identity-providers versions = all
18422024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/identity-providers/local/users versions = all
18432024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/system/identity-providers/local/users/{user_id} versions = all
18442024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/identity-providers/local/users/{user_id}/set-password versions = all
18452024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/identity-providers/saml versions = all
18462024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/identity-providers/saml/{provider} versions = all
18472024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/ip-pools versions = all
18482024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/ip-pools versions = all
18492024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/system/ip-pools/{pool} versions = all
18502024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/ip-pools/{pool} versions = all
18512024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/system/ip-pools/{pool} versions = all
18522024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/ip-pools/{pool}/ranges versions = all
18532024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/ip-pools/{pool}/ranges/add versions = all
18542024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/ip-pools/{pool}/ranges/remove versions = all
18552024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/ip-pools/{pool}/silos versions = all
18562024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/ip-pools/{pool}/silos versions = all
18572024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/system/ip-pools/{pool}/silos/{silo} versions = all
18582024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/system/ip-pools/{pool}/silos/{silo} versions = all
18592024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/ip-pools/{pool}/utilization versions = all
18602024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/ip-pools-service versions = all
18612024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/ip-pools-service/ranges versions = all
18622024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/ip-pools-service/ranges/add versions = all
18632024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/ip-pools-service/ranges/remove versions = all
18642024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/metrics/{metric_name} versions = all
18652024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/networking/address-lot versions = all
18662024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/networking/address-lot versions = all
18672024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/system/networking/address-lot/{address_lot} versions = all
18682024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/networking/address-lot/{address_lot}/blocks versions = all
18692024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/networking/allow-list versions = all
18702024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/system/networking/allow-list versions = all
18712024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/networking/bfd-disable versions = all
18722024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/networking/bfd-enable versions = all
18732024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/networking/bfd-status versions = all
18742024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/system/networking/bgp versions = all
18752024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/networking/bgp versions = all
18762024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/networking/bgp versions = all
18772024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/networking/bgp-announce-set versions = all
18782024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/system/networking/bgp-announce-set versions = all
18792024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/system/networking/bgp-announce-set/{announce_set} versions = all
18802024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/networking/bgp-announce-set/{announce_set}/announcement versions = all
18812024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/networking/bgp-exported versions = all
18822024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/networking/bgp-message-history versions = all
18832024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/networking/bgp-routes-ipv4 versions = all
18842024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/networking/bgp-status versions = all
18852024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/networking/loopback-address versions = all
18862024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/networking/loopback-address versions = all
18872024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/system/networking/loopback-address/{rack_id}/{switch_location}/{address}/{subnet_mask} versions = all
18882024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/system/networking/switch-port-settings versions = all
18892024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/networking/switch-port-settings versions = all
18902024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/networking/switch-port-settings versions = all
18912024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/networking/switch-port-settings/{port} versions = all
18922024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/policy versions = all
18932024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/system/policy versions = all
18942024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/roles versions = all
18952024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/roles/{role_name} versions = all
18962024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/silo-quotas versions = all
18972024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/silos versions = all
18982024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/silos versions = all
18992024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/system/silos/{silo} versions = all
19002024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/silos/{silo} versions = all
19012024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/silos/{silo}/ip-pools versions = all
19022024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/silos/{silo}/policy versions = all
19032024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/system/silos/{silo}/policy versions = all
19042024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/silos/{silo}/quotas versions = all
19052024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/system/silos/{silo}/quotas versions = all
19062024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/system/timeseries/query versions = all
19072024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/timeseries/schemas versions = all
19082024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/system/update/repository versions = all
19092024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/update/repository/{system_version} versions = all
19102024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/users versions = all
19112024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/users/{user_id} versions = all
19122024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/users-builtin versions = all
19132024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/users-builtin/{user} versions = all
19142024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/utilization/silos versions = all
19152024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/system/utilization/silos/{silo} versions = all
19162024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/users versions = all
19172024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/utilization versions = all
19182024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/vpc-firewall-rules versions = all
19192024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/vpc-firewall-rules versions = all
19202024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/vpc-router-routes versions = all
19212024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/vpc-router-routes versions = all
19222024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/vpc-router-routes/{route} versions = all
19232024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/vpc-router-routes/{route} versions = all
19242024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/vpc-router-routes/{route} versions = all
19252024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/vpc-routers versions = all
19262024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/vpc-routers versions = all
19272024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/vpc-routers/{router} versions = all
19282024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/vpc-routers/{router} versions = all
19292024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/vpc-routers/{router} versions = all
19302024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/vpc-subnets versions = all
19312024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/vpc-subnets versions = all
19322024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/vpc-subnets/{subnet} versions = all
19332024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/vpc-subnets/{subnet} versions = all
19342024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/vpc-subnets/{subnet} versions = all
19352024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/vpc-subnets/{subnet}/network-interfaces versions = all
19362024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/vpcs versions = all
19372024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = POST path = /v1/vpcs versions = all
19382024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = DELETE path = /v1/vpcs/{vpc} versions = all
19392024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = GET path = /v1/vpcs/{vpc} versions = all
19402024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): registered endpoint local_addr = 172.30.2.7:443 method = PUT path = /v1/vpcs/{vpc} versions = all
19412024-11-21T19:34:18.109ZINFOnexus (dropshot_external): listening file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327 local_addr = 172.30.2.7:443
19422024-11-21T19:34:18.109ZDEBGnexus (dropshot_external): successfully registered DTrace USDT probes local_addr = 172.30.2.7:443
19432024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = / versions = all
19442024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /access versions = all
19452024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /assets/{path} versions = all
19462024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /device/auth versions = all
19472024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /device/confirm versions = all
19482024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /device/success versions = all
19492024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /device/token versions = all
19502024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /device/verify versions = all
19512024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /experimental/v1/probes versions = all
19522024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /experimental/v1/probes versions = all
19532024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /experimental/v1/probes/{probe} versions = all
19542024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /experimental/v1/probes/{probe} versions = all
19552024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /images versions = all
19562024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /login versions = all
19572024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /login/{silo_name}/local versions = all
19582024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /login/{silo_name}/saml/{provider_name} versions = all
19592024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /login/{silo_name}/saml/{provider_name} versions = all
19602024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /login/{silo_name}/saml/{provider_name}/redirect versions = all
19612024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /lookup/{path} versions = all
19622024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /projects/{path} versions = all
19632024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /projects-new versions = all
19642024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /settings/{path} versions = all
19652024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /system/{path} versions = all
19662024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /utilization versions = all
19672024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/certificates versions = all
19682024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/certificates versions = all
19692024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/certificates/{certificate} versions = all
19702024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/certificates/{certificate} versions = all
19712024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/disks versions = all
19722024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/disks versions = all
19732024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/disks/{disk} versions = all
19742024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/disks/{disk} versions = all
19752024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/disks/{disk}/bulk-write versions = all
19762024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/disks/{disk}/bulk-write-start versions = all
19772024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/disks/{disk}/bulk-write-stop versions = all
19782024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/disks/{disk}/finalize versions = all
19792024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/disks/{disk}/metrics/{metric} versions = all
19802024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/floating-ips versions = all
19812024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/floating-ips versions = all
19822024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/floating-ips/{floating_ip} versions = all
19832024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/floating-ips/{floating_ip} versions = all
19842024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/floating-ips/{floating_ip} versions = all
19852024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/floating-ips/{floating_ip}/attach versions = all
19862024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/floating-ips/{floating_ip}/detach versions = all
19872024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/groups versions = all
19882024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/groups/{group_id} versions = all
19892024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/images versions = all
19902024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/images versions = all
19912024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/images/{image} versions = all
19922024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/images/{image} versions = all
19932024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/images/{image}/demote versions = all
19942024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/images/{image}/promote versions = all
19952024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/instances versions = all
19962024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/instances versions = all
19972024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/instances/{instance} versions = all
19982024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/instances/{instance} versions = all
19992024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/instances/{instance} versions = all
20002024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/instances/{instance}/disks versions = all
20012024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/instances/{instance}/disks/attach versions = all
20022024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/instances/{instance}/disks/detach versions = all
20032024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/instances/{instance}/external-ips versions = all
20042024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/instances/{instance}/external-ips/ephemeral versions = all
20052024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/instances/{instance}/external-ips/ephemeral versions = all
20062024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/instances/{instance}/reboot versions = all
20072024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/instances/{instance}/serial-console versions = all
20082024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/instances/{instance}/serial-console/stream versions = all
20092024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/instances/{instance}/ssh-public-keys versions = all
20102024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/instances/{instance}/start versions = all
20112024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/instances/{instance}/stop versions = all
20122024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/internet-gateway-ip-addresses versions = all
20132024-11-21T19:34:18.115ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/internet-gateway-ip-addresses versions = all
20142024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/internet-gateway-ip-addresses/{address} versions = all
20152024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/internet-gateway-ip-pools versions = all
20162024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/internet-gateway-ip-pools versions = all
20172024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/internet-gateway-ip-pools/{pool} versions = all
20182024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/internet-gateways versions = all
20192024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/internet-gateways versions = all
20202024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/internet-gateways/{gateway} versions = all
20212024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/internet-gateways/{gateway} versions = all
20222024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/ip-pools versions = all
20232024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/ip-pools/{pool} versions = all
20242024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/login/{silo_name}/local versions = all
20252024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/logout versions = all
20262024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/me versions = all
20272024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/me/groups versions = all
20282024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/me/ssh-keys versions = all
20292024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/me/ssh-keys versions = all
20302024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/me/ssh-keys/{ssh_key} versions = all
20312024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/me/ssh-keys/{ssh_key} versions = all
20322024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/metrics/{metric_name} versions = all
20332024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/network-interfaces versions = all
20342024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/network-interfaces versions = all
20352024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/network-interfaces/{interface} versions = all
20362024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/network-interfaces/{interface} versions = all
20372024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/network-interfaces/{interface} versions = all
20382024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/ping versions = all
20392024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/policy versions = all
20402024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/policy versions = all
20412024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/projects versions = all
20422024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/projects versions = all
20432024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/projects/{project} versions = all
20442024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/projects/{project} versions = all
20452024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/projects/{project} versions = all
20462024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/projects/{project}/policy versions = all
20472024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/projects/{project}/policy versions = all
20482024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/snapshots versions = all
20492024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/snapshots versions = all
20502024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/snapshots/{snapshot} versions = all
20512024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/snapshots/{snapshot} versions = all
20522024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/hardware/disks versions = all
20532024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/hardware/disks/{disk_id} versions = all
20542024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/hardware/racks versions = all
20552024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/hardware/racks/{rack_id} versions = all
20562024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/hardware/sleds versions = all
20572024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/hardware/sleds versions = all
20582024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/hardware/sleds/{sled_id} versions = all
20592024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/hardware/sleds/{sled_id}/disks versions = all
20602024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/hardware/sleds/{sled_id}/instances versions = all
20612024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/system/hardware/sleds/{sled_id}/provision-policy versions = all
20622024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/hardware/sleds-uninitialized versions = all
20632024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/hardware/switch-port versions = all
20642024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/system/hardware/switch-port/{port}/settings versions = all
20652024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/hardware/switch-port/{port}/settings versions = all
20662024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/hardware/switch-port/{port}/status versions = all
20672024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/hardware/switches versions = all
20682024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/hardware/switches/{switch_id} versions = all
20692024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/identity-providers versions = all
20702024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/identity-providers/local/users versions = all
20712024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/system/identity-providers/local/users/{user_id} versions = all
20722024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/identity-providers/local/users/{user_id}/set-password versions = all
20732024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/identity-providers/saml versions = all
20742024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/identity-providers/saml/{provider} versions = all
20752024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/ip-pools versions = all
20762024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/ip-pools versions = all
20772024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/system/ip-pools/{pool} versions = all
20782024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/ip-pools/{pool} versions = all
20792024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/system/ip-pools/{pool} versions = all
20802024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/ip-pools/{pool}/ranges versions = all
20812024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/ip-pools/{pool}/ranges/add versions = all
20822024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/ip-pools/{pool}/ranges/remove versions = all
20832024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/ip-pools/{pool}/silos versions = all
20842024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/ip-pools/{pool}/silos versions = all
20852024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/system/ip-pools/{pool}/silos/{silo} versions = all
20862024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/system/ip-pools/{pool}/silos/{silo} versions = all
20872024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/ip-pools/{pool}/utilization versions = all
20882024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/ip-pools-service versions = all
20892024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/ip-pools-service/ranges versions = all
20902024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/ip-pools-service/ranges/add versions = all
20912024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/ip-pools-service/ranges/remove versions = all
20922024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/metrics/{metric_name} versions = all
20932024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/networking/address-lot versions = all
20942024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/networking/address-lot versions = all
20952024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/system/networking/address-lot/{address_lot} versions = all
20962024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/networking/address-lot/{address_lot}/blocks versions = all
20972024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/networking/allow-list versions = all
20982024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/system/networking/allow-list versions = all
20992024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/networking/bfd-disable versions = all
21002024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/networking/bfd-enable versions = all
21012024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/networking/bfd-status versions = all
21022024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/system/networking/bgp versions = all
21032024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/networking/bgp versions = all
21042024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/networking/bgp versions = all
21052024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/networking/bgp-announce-set versions = all
21062024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/system/networking/bgp-announce-set versions = all
21072024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/system/networking/bgp-announce-set/{announce_set} versions = all
21082024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/networking/bgp-announce-set/{announce_set}/announcement versions = all
21092024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/networking/bgp-exported versions = all
21102024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/networking/bgp-message-history versions = all
21112024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/networking/bgp-routes-ipv4 versions = all
21122024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/networking/bgp-status versions = all
21132024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/networking/loopback-address versions = all
21142024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/networking/loopback-address versions = all
21152024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/system/networking/loopback-address/{rack_id}/{switch_location}/{address}/{subnet_mask} versions = all
21162024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/system/networking/switch-port-settings versions = all
21172024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/networking/switch-port-settings versions = all
21182024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/networking/switch-port-settings versions = all
21192024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/networking/switch-port-settings/{port} versions = all
21202024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/policy versions = all
21212024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/system/policy versions = all
21222024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/roles versions = all
21232024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/roles/{role_name} versions = all
21242024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/silo-quotas versions = all
21252024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/silos versions = all
21262024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/silos versions = all
21272024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/system/silos/{silo} versions = all
21282024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/silos/{silo} versions = all
21292024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/silos/{silo}/ip-pools versions = all
21302024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/silos/{silo}/policy versions = all
21312024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/system/silos/{silo}/policy versions = all
21322024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/silos/{silo}/quotas versions = all
21332024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/system/silos/{silo}/quotas versions = all
21342024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/system/timeseries/query versions = all
21352024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/timeseries/schemas versions = all
21362024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/system/update/repository versions = all
21372024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/update/repository/{system_version} versions = all
21382024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/users versions = all
21392024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/users/{user_id} versions = all
21402024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/users-builtin versions = all
21412024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/users-builtin/{user} versions = all
21422024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/utilization/silos versions = all
21432024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/system/utilization/silos/{silo} versions = all
21442024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/users versions = all
21452024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/utilization versions = all
21462024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/vpc-firewall-rules versions = all
21472024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/vpc-firewall-rules versions = all
21482024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/vpc-router-routes versions = all
21492024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/vpc-router-routes versions = all
21502024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/vpc-router-routes/{route} versions = all
21512024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/vpc-router-routes/{route} versions = all
21522024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/vpc-router-routes/{route} versions = all
21532024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/vpc-routers versions = all
21542024-11-21T19:34:18.116ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/vpc-routers versions = all
21552024-11-21T19:34:18.117ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/vpc-routers/{router} versions = all
21562024-11-21T19:34:18.117ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/vpc-routers/{router} versions = all
21572024-11-21T19:34:18.117ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/vpc-routers/{router} versions = all
21582024-11-21T19:34:18.117ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/vpc-subnets versions = all
21592024-11-21T19:34:18.117ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/vpc-subnets versions = all
21602024-11-21T19:34:18.117ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/vpc-subnets/{subnet} versions = all
21612024-11-21T19:34:18.117ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/vpc-subnets/{subnet} versions = all
21622024-11-21T19:34:18.117ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/vpc-subnets/{subnet} versions = all
21632024-11-21T19:34:18.117ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/vpc-subnets/{subnet}/network-interfaces versions = all
21642024-11-21T19:34:18.117ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/vpcs versions = all
21652024-11-21T19:34:18.117ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = POST path = /v1/vpcs versions = all
21662024-11-21T19:34:18.117ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = DELETE path = /v1/vpcs/{vpc} versions = all
21672024-11-21T19:34:18.117ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = GET path = /v1/vpcs/{vpc} versions = all
21682024-11-21T19:34:18.117ZDEBGnexus (dropshot_external_techport): registered endpoint local_addr = [fd00:1122:3344:101::c]:12228 method = PUT path = /v1/vpcs/{vpc} versions = all
21692024-11-21T19:34:18.117ZINFOnexus (dropshot_external_techport): listening file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327 local_addr = [fd00:1122:3344:101::c]:12228
21702024-11-21T19:34:18.117ZDEBGnexus (dropshot_external_techport): successfully registered DTrace USDT probes local_addr = [fd00:1122:3344:101::c]:12228
21712024-11-21T19:34:18.123ZDEBGnexus (nexus-producer-server): registered DTrace probes
21722024-11-21T19:34:18.128ZDEBGnexus (dropshot): registered endpoint local_addr = [fd00:1122:3344:101::c]:46619 method = GET path = /{producer_id} versions = all
21732024-11-21T19:34:18.129ZINFOnexus (dropshot): listening file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327 local_addr = [fd00:1122:3344:101::c]:46619
21742024-11-21T19:34:18.129ZDEBGnexus (dropshot): successfully registered DTrace USDT probes local_addr = [fd00:1122:3344:101::c]:46619
21752024-11-21T19:34:18.129ZDEBGnexus (nexus-producer-server): Requested any available port, Dropshot server has been bound to [fd00:1122:3344:101::c]:46619
21762024-11-21T19:34:18.129ZDEBGnexus (nexus-producer-server): Nexus IP provided explicitly, will use it addr = [fd00:1122:3344:101::c]:12221
21772024-11-21T19:34:18.129ZDEBGnexus (nexus-producer-server): starting producer registration task
21782024-11-21T19:34:18.131ZINFOnexus (nexus-producer-server): starting oximeter metric producer server address = [fd00:1122:3344:101::c]:46619 file = oximeter/producer/src/lib.rs:279 interval = 10s producer_id = 915e11b8-d7cb-4730-9906-7980540813dc
21792024-11-21T19:34:18.131ZDEBGnexus (producer-registration-task): registering / renewing oximeter producer lease with Nexus
21802024-11-21T19:34:18.131ZDEBGnexus (producer-registration-task): using nexus address for registration addr = [fd00:1122:3344:101::c]:12221
21812024-11-21T19:34:18.161ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
21822024-11-21T19:34:18.162ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "1c7909f9-f2db-4e07-8130-a68c5e0e5fd0", "content-length": "1", "date": "Thu, 21 Nov 2024 19:34:17 GMT"} })
21832024-11-21T19:34:18.163ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
21842024-11-21T19:34:18.163ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
21852024-11-21T19:34:18.166ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 337.158765ms iteration = 3 status = null
21862024-11-21T19:34:18.214ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
21872024-11-21T19:34:18.218ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 389.248601ms iteration = 3 status = {}
21882024-11-21T19:34:18.251ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
21892024-11-21T19:34:18.251ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
21902024-11-21T19:34:18.251ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "afbcdda5-e905-4aec-a5a4-9008ad157414", "content-length": "372", "date": "Thu, 21 Nov 2024 19:34:18 GMT"} })
21912024-11-21T19:34:18.251ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
21922024-11-21T19:34:18.252ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
21932024-11-21T19:34:18.253ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
21942024-11-21T19:34:18.266ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
21952024-11-21T19:34:18.267ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "cdb3b10e-473c-46b8-a0ac-1ed07ac17beb", "content-length": "691", "date": "Thu, 21 Nov 2024 19:34:18 GMT"} })
21962024-11-21T19:34:18.267ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
21972024-11-21T19:34:18.267ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
21982024-11-21T19:34:18.267ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
21992024-11-21T19:34:18.268ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 439.205719ms iteration = 3 status = {}
22002024-11-21T19:34:18.284ZDEBGnexus (producer-registration-task): client request body = Some(Body) method = POST uri = http://[fd00:1122:3344:101::c]:12221/metrics/producers
22012024-11-21T19:34:18.284ZINFOnexus (dropshot_internal): accepted connection file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023 local_addr = [fd00:1122:3344:101::c]:12221 remote_addr = [fd00:1122:3344:101::c]:64275
22022024-11-21T19:34:18.299ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
22032024-11-21T19:34:18.299ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
22042024-11-21T19:34:18.299ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "2011b11e-d4e2-43d0-ba97-6fbe89890dfe", "date": "Thu, 21 Nov 2024 19:34:18 GMT"} })
22052024-11-21T19:34:18.299ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
22062024-11-21T19:34:18.299ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
22072024-11-21T19:34:18.302ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
22082024-11-21T19:34:18.302ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
22092024-11-21T19:34:18.306ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
22102024-11-21T19:34:18.306ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
22112024-11-21T19:34:18.308ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:64275 req_id = 1e28d36a-303d-4891-908e-d333617e0da3 roles = RoleSet { roles: {} } uri = /metrics/producers
22122024-11-21T19:34:18.308ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:64275 req_id = 1e28d36a-303d-4891-908e-d333617e0da3 resource = Database result = Ok(()) uri = /metrics/producers
22132024-11-21T19:34:18.310ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
22142024-11-21T19:34:18.310ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
22152024-11-21T19:34:18.310ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "cf01e285-4b46-4030-bb78-646fc2e45ff6", "date": "Thu, 21 Nov 2024 19:34:18 GMT"} })
22162024-11-21T19:34:18.312ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 483.213966ms iteration = 3 status = {}
22172024-11-21T19:34:18.337ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
22182024-11-21T19:34:18.337ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::9]:5353/config", status: 204, headers: {"x-request-id": "22112e9a-6244-4bd4-9c92-a8ea14172d97", "date": "Thu, 21 Nov 2024 19:34:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
22192024-11-21T19:34:18.337ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
22202024-11-21T19:34:18.339ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::8]:5353/config", status: 204, headers: {"x-request-id": "c95991d1-45ed-4657-807b-b19d47faf8f8", "date": "Thu, 21 Nov 2024 19:34:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
22212024-11-21T19:34:18.339ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
22222024-11-21T19:34:18.340ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
22232024-11-21T19:34:18.340ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 303.386214ms iteration = 4 status = {"generation":2,"server_results":{"[fd00:1122:3344:101::8]:5353":{"Ok":null},"[fd00:1122:3344:101::9]:5353":{"Ok":null}}}
22242024-11-21T19:34:18.349ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
22252024-11-21T19:34:18.355ZINFOnexus: assigned collector to new producer collector_id = 5fb99117-c685-44d5-bec2-c58adb63b619 file = nexus/src/app/oximeter.rs:76 producer_id = 915e11b8-d7cb-4730-9906-7980540813dc
22262024-11-21T19:34:18.360ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 72858 local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:64275 req_id = 1e28d36a-303d-4891-908e-d333617e0da3 response_code = 201 uri = /metrics/producers
22272024-11-21T19:34:18.360ZDEBGnexus (producer-registration-task): client response result = Ok(Response { url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", status: 201, headers: {"content-type": "application/json", "x-request-id": "1e28d36a-303d-4891-908e-d333617e0da3", "content-length": "41", "date": "Thu, 21 Nov 2024 19:34:18 GMT"} })
22282024-11-21T19:34:18.362ZDEBGnexus (producer-registration-task): registered with nexus successfully
22292024-11-21T19:34:18.362ZDEBGnexus (producer-registration-task): pausing until time to renew lease lease_duration = 600s wait_period = 150s
22302024-11-21T19:34:18.375ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
22312024-11-21T19:34:18.376ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
22322024-11-21T19:34:18.380ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
22332024-11-21T19:34:18.380ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
22342024-11-21T19:34:18.380ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
22352024-11-21T19:34:18.389ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
22362024-11-21T19:34:18.389ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "796e779c-da28-4b83-b532-9d5713d02be8", "content-length": "134", "date": "Thu, 21 Nov 2024 19:34:18 GMT"} })
22372024-11-21T19:34:18.389ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
22382024-11-21T19:34:18.389ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
22392024-11-21T19:34:18.389ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
22402024-11-21T19:34:18.390ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
22412024-11-21T19:34:18.390ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
22422024-11-21T19:34:18.390ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
22432024-11-21T19:34:18.390ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
22442024-11-21T19:34:18.390ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
22452024-11-21T19:34:18.390ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
22462024-11-21T19:34:18.390ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
22472024-11-21T19:34:18.390ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
22482024-11-21T19:34:18.390ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
22492024-11-21T19:34:18.390ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
22502024-11-21T19:34:18.392ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
22512024-11-21T19:34:18.392ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "54d84c54-7464-4ae9-9c37-b64f70a7da48", "content-length": "2", "date": "Thu, 21 Nov 2024 19:34:18 GMT"} })
22522024-11-21T19:34:18.392ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
22532024-11-21T19:34:18.392ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "98f809cd-d495-4ea5-bcef-6b6e3c85b54a", "content-length": "51", "date": "Thu, 21 Nov 2024 19:34:18 GMT"} })
22542024-11-21T19:34:18.392ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
22552024-11-21T19:34:18.392ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
22562024-11-21T19:34:18.393ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
22572024-11-21T19:34:18.394ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:1::1]:5353/config", status: 204, headers: {"x-request-id": "a24e8ea1-e48b-4d56-84e4-67a839f581e2", "date": "Thu, 21 Nov 2024 19:34:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
22582024-11-21T19:34:18.394ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
22592024-11-21T19:34:18.396ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:3::1]:5353/config", status: 204, headers: {"x-request-id": "004bbbe0-7edf-4ebb-ba4b-8c6fe5b50a3e", "date": "Thu, 21 Nov 2024 19:34:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
22602024-11-21T19:34:18.396ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
22612024-11-21T19:34:18.397ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:2::1]:5353/config", status: 204, headers: {"x-request-id": "75da9ede-0ae7-4850-92d8-2cdff789637d", "date": "Thu, 21 Nov 2024 19:34:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
22622024-11-21T19:34:18.397ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
22632024-11-21T19:34:18.398ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
22642024-11-21T19:34:18.398ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 358.781594ms iteration = 4 status = {"generation":1,"server_results":{"[fd00:1122:3344:1::1]:5353":{"Ok":null},"[fd00:1122:3344:2::1]:5353":{"Ok":null},"[fd00:1122:3344:3::1]:5353":{"Ok":null}}}
22652024-11-21T19:34:18.486ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "62daf209-b2ad-449c-aff5-48e558d7b195", "date": "Thu, 21 Nov 2024 19:34:18 GMT"} })
22662024-11-21T19:34:18.486ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
22672024-11-21T19:34:18.486ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
22682024-11-21T19:34:18.487ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "05f40110-30b6-46c0-a351-39ac81651d48", "content-length": "682", "date": "Thu, 21 Nov 2024 19:34:18 GMT"} })
22692024-11-21T19:34:18.487ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
22702024-11-21T19:34:18.487ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
22712024-11-21T19:34:18.503ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
22722024-11-21T19:34:18.504ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
22732024-11-21T19:34:18.507ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
22742024-11-21T19:34:18.507ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
22752024-11-21T19:34:18.512ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
22762024-11-21T19:34:18.512ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
22772024-11-21T19:34:18.518ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 689.217589ms iteration = 3 status = {}
22782024-11-21T19:34:27.828ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 8 reason = Timeout
22792024-11-21T19:34:27.828ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
22802024-11-21T19:34:27.828ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
22812024-11-21T19:34:27.829ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 8 reason = Timeout
22822024-11-21T19:34:27.829ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
22832024-11-21T19:34:27.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
22842024-11-21T19:34:27.830ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
22852024-11-21T19:34:27.831ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
22862024-11-21T19:34:27.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
22872024-11-21T19:34:27.833ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 3.800081ms iteration = 8 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
22882024-11-21T19:34:27.833ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
22892024-11-21T19:34:27.833ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
22902024-11-21T19:34:27.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
22912024-11-21T19:34:27.835ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
22922024-11-21T19:34:27.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
22932024-11-21T19:34:27.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
22942024-11-21T19:34:27.839ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
22952024-11-21T19:34:27.839ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
22962024-11-21T19:34:27.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
22972024-11-21T19:34:27.853ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
22982024-11-21T19:34:27.853ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 25.592791ms iteration = 8 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
22992024-11-21T19:34:32.029ZINFOnexus (dropshot): accepted connection file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023 local_addr = [fd00:1122:3344:101::c]:46619 remote_addr = [fd00:1122:3344:101::d]:56113
23002024-11-21T19:34:32.038ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 8654 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = fac14bd0-36c8-4b8f-9f3f-a303715d1cfc response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
23012024-11-21T19:34:37.829ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 9 reason = Timeout
23022024-11-21T19:34:37.829ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
23032024-11-21T19:34:37.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
23042024-11-21T19:34:37.829ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 9 reason = Timeout
23052024-11-21T19:34:37.829ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
23062024-11-21T19:34:37.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
23072024-11-21T19:34:37.831ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
23082024-11-21T19:34:37.834ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
23092024-11-21T19:34:37.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
23102024-11-21T19:34:37.834ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
23112024-11-21T19:34:37.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
23122024-11-21T19:34:37.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
23132024-11-21T19:34:37.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
23142024-11-21T19:34:37.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
23152024-11-21T19:34:37.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
23162024-11-21T19:34:37.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
23172024-11-21T19:34:37.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
23182024-11-21T19:34:37.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
23192024-11-21T19:34:37.852ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
23202024-11-21T19:34:37.852ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 23.379252ms iteration = 9 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
23212024-11-21T19:34:37.868ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 38.281351ms iteration = 9 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
23222024-11-21T19:34:42.030ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 222 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 774401f5-e29b-4831-9a3d-d534a8e0baca response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
23232024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 4 reason = Timeout
23242024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
23252024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 10 reason = Timeout
23262024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 4 reason = Timeout
23272024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
23282024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
23292024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 4 reason = Timeout
23302024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 4 reason = Timeout
23312024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
23322024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
23332024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 4 reason = Timeout
23342024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
23352024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 5 reason = Timeout
23362024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
23372024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 4 reason = Timeout
23382024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 4 reason = Timeout
23392024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
23402024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
23412024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
23422024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
23432024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
23442024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
23452024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
23462024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
23472024-11-21T19:34:47.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
23482024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 10 reason = Timeout
23492024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 4 reason = Timeout
23502024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
23512024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
23522024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 4 reason = Timeout
23532024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
23542024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 4 reason = Timeout
23552024-11-21T19:34:47.830ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
23562024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
23572024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 4 reason = Timeout
23582024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
23592024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
23602024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 4 reason = Timeout
23612024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
23622024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 4 reason = Timeout
23632024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
23642024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 4 reason = Timeout
23652024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
23662024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
23672024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
23682024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
23692024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
23702024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
23712024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
23722024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
23732024-11-21T19:34:47.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
23742024-11-21T19:34:47.833ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 830, tv_nsec: 712674776 } })
23752024-11-21T19:34:47.833ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 830, tv_nsec: 712894579 } })
23762024-11-21T19:34:47.833ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
23772024-11-21T19:34:47.833ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
23782024-11-21T19:34:47.834ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 5.167659ms iteration = 4 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
23792024-11-21T19:34:47.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
23802024-11-21T19:34:47.835ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
23812024-11-21T19:34:47.835ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
23822024-11-21T19:34:47.837ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
23832024-11-21T19:34:47.837ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
23842024-11-21T19:34:47.837ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
23852024-11-21T19:34:47.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
23862024-11-21T19:34:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
23872024-11-21T19:34:47.838ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
23882024-11-21T19:34:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
23892024-11-21T19:34:47.838ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
23902024-11-21T19:34:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
23912024-11-21T19:34:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
23922024-11-21T19:34:47.840ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
23932024-11-21T19:34:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
23942024-11-21T19:34:47.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
23952024-11-21T19:34:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
23962024-11-21T19:34:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
23972024-11-21T19:34:47.842ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
23982024-11-21T19:34:47.843ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 13.027444ms iteration = 4 status = {"errors":[],"garbage_collect_requested":[]}
23992024-11-21T19:34:47.844ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
24002024-11-21T19:34:47.844ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
24012024-11-21T19:34:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
24022024-11-21T19:34:47.848ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
24032024-11-21T19:34:47.848ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 830, tv_nsec: 728270354 } })
24042024-11-21T19:34:47.848ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
24052024-11-21T19:34:47.849ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
24062024-11-21T19:34:47.849ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
24072024-11-21T19:34:47.849ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
24082024-11-21T19:34:47.851ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
24092024-11-21T19:34:47.855ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
24102024-11-21T19:34:47.855ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
24112024-11-21T19:34:47.855ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
24122024-11-21T19:34:47.872ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
24132024-11-21T19:34:47.874ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
24142024-11-21T19:34:47.874ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
24152024-11-21T19:34:47.875ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
24162024-11-21T19:34:47.877ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
24172024-11-21T19:34:47.877ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
24182024-11-21T19:34:47.878ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
24192024-11-21T19:34:47.880ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
24202024-11-21T19:34:47.882ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
24212024-11-21T19:34:47.883ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 53.800789ms iteration = 4 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
24222024-11-21T19:34:47.885ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
24232024-11-21T19:34:47.885ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
24242024-11-21T19:34:47.885ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
24252024-11-21T19:34:47.886ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
24262024-11-21T19:34:47.886ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 57.124483ms iteration = 10 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
24272024-11-21T19:34:47.887ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 56.783608ms iteration = 4 status = {"errors":[],"records_set_to_done":[]}
24282024-11-21T19:34:47.891ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 60.928424ms iteration = 4 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
24292024-11-21T19:34:47.891ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
24302024-11-21T19:34:47.891ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
24312024-11-21T19:34:47.891ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
24322024-11-21T19:34:47.891ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
24332024-11-21T19:34:47.891ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
24342024-11-21T19:34:47.891ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
24352024-11-21T19:34:47.892ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
24362024-11-21T19:34:47.892ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
24372024-11-21T19:34:47.892ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
24382024-11-21T19:34:47.893ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
24392024-11-21T19:34:47.894ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
24402024-11-21T19:34:47.894ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
24412024-11-21T19:34:47.894ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
24422024-11-21T19:34:47.894ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 64.203518ms iteration = 4 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
24432024-11-21T19:34:47.895ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 65.05356ms iteration = 10 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
24442024-11-21T19:34:47.895ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
24452024-11-21T19:34:47.896ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
24462024-11-21T19:34:47.896ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
24472024-11-21T19:34:47.896ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 65.692858ms iteration = 4 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
24482024-11-21T19:34:47.896ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
24492024-11-21T19:34:47.896ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
24502024-11-21T19:34:47.896ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
24512024-11-21T19:34:47.898ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
24522024-11-21T19:34:47.898ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
24532024-11-21T19:34:47.898ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
24542024-11-21T19:34:47.900ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
24552024-11-21T19:34:47.900ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
24562024-11-21T19:34:47.900ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
24572024-11-21T19:34:47.900ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
24582024-11-21T19:34:47.904ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
24592024-11-21T19:34:47.904ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
24602024-11-21T19:34:47.904ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
24612024-11-21T19:34:47.906ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 77.012329ms iteration = 4 status = 0
24622024-11-21T19:34:47.906ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
24632024-11-21T19:34:47.906ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
24642024-11-21T19:34:47.907ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
24652024-11-21T19:34:47.910ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
24662024-11-21T19:34:47.910ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
24672024-11-21T19:34:47.910ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
24682024-11-21T19:34:47.930ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
24692024-11-21T19:34:47.930ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
24702024-11-21T19:34:47.930ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 100.377961ms iteration = 4 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
24712024-11-21T19:34:47.933ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
24722024-11-21T19:34:47.933ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
24732024-11-21T19:34:47.934ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "8ddb4ed2-40dd-47b3-accc-9fd58faf1b95", "content-length": "26", "date": "Thu, 21 Nov 2024 19:34:47 GMT"} })
24742024-11-21T19:34:47.934ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
24752024-11-21T19:34:47.934ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
24762024-11-21T19:34:47.934ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
24772024-11-21T19:34:47.934ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "af510644-451c-4e13-9868-9063a7018278", "content-length": "26", "date": "Thu, 21 Nov 2024 19:34:47 GMT"} })
24782024-11-21T19:34:47.934ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
24792024-11-21T19:34:47.936ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
24802024-11-21T19:34:47.936ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
24812024-11-21T19:34:47.949ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
24822024-11-21T19:34:47.949ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
24832024-11-21T19:34:47.949ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "bdfb075a-1971-433e-9098-c7be2dbaba09", "content-length": "26", "date": "Thu, 21 Nov 2024 19:34:47 GMT"} })
24842024-11-21T19:34:47.949ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
24852024-11-21T19:34:47.950ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
24862024-11-21T19:34:47.983ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 154.506044ms iteration = 5 status = {"physical_disks_added":0}
24872024-11-21T19:34:48.000ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
24882024-11-21T19:34:48.000ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
24892024-11-21T19:34:48.001ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
24902024-11-21T19:34:48.003ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
24912024-11-21T19:34:48.003ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
24922024-11-21T19:34:48.003ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
24932024-11-21T19:34:48.022ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
24942024-11-21T19:34:48.022ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 193.04423ms iteration = 4 status = {}
24952024-11-21T19:34:48.022ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "8dcff7a6-fe33-481b-b64f-21350020a985", "content-length": "1", "date": "Thu, 21 Nov 2024 19:34:47 GMT"} })
24962024-11-21T19:34:48.024ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
24972024-11-21T19:34:48.024ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
24982024-11-21T19:34:48.026ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 197.60908ms iteration = 4 status = null
24992024-11-21T19:34:48.040ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "8bdc86c4-2bf5-4f91-8236-8a190046984c", "content-length": "372", "date": "Thu, 21 Nov 2024 19:34:48 GMT"} })
25002024-11-21T19:34:48.040ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
25012024-11-21T19:34:48.040ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
25022024-11-21T19:34:48.055ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
25032024-11-21T19:34:48.055ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
25042024-11-21T19:34:48.056ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "8e1fb530-8c5d-4ebd-8e07-8a4b4d26a1c5", "date": "Thu, 21 Nov 2024 19:34:48 GMT"} })
25052024-11-21T19:34:48.056ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
25062024-11-21T19:34:48.056ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
25072024-11-21T19:34:48.059ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
25082024-11-21T19:34:48.059ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
25092024-11-21T19:34:48.062ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
25102024-11-21T19:34:48.063ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
25112024-11-21T19:34:48.117ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
25122024-11-21T19:34:48.117ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
25132024-11-21T19:34:48.120ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
25142024-11-21T19:34:48.121ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "1396a448-fd5b-42a0-9235-f400859bd0c7", "content-length": "691", "date": "Thu, 21 Nov 2024 19:34:48 GMT"} })
25152024-11-21T19:34:48.121ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
25162024-11-21T19:34:48.121ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
25172024-11-21T19:34:48.121ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
25182024-11-21T19:34:48.122ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 293.288029ms iteration = 4 status = {}
25192024-11-21T19:34:48.123ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
25202024-11-21T19:34:48.123ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
25212024-11-21T19:34:48.123ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
25222024-11-21T19:34:48.124ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
25232024-11-21T19:34:48.124ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
25242024-11-21T19:34:48.125ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "259659ff-7707-404c-9735-3a35501782ca", "date": "Thu, 21 Nov 2024 19:34:48 GMT"} })
25252024-11-21T19:34:48.126ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 296.171127ms iteration = 4 status = {}
25262024-11-21T19:34:48.129ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
25272024-11-21T19:34:48.129ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "8d21ba57-f28d-4778-b838-5a625a4b2fa6", "content-length": "134", "date": "Thu, 21 Nov 2024 19:34:47 GMT"} })
25282024-11-21T19:34:48.129ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
25292024-11-21T19:34:48.129ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
25302024-11-21T19:34:48.129ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
25312024-11-21T19:34:48.129ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
25322024-11-21T19:34:48.129ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
25332024-11-21T19:34:48.129ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
25342024-11-21T19:34:48.129ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
25352024-11-21T19:34:48.129ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
25362024-11-21T19:34:48.130ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
25372024-11-21T19:34:48.130ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
25382024-11-21T19:34:48.130ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
25392024-11-21T19:34:48.130ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
25402024-11-21T19:34:48.130ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
25412024-11-21T19:34:48.131ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
25422024-11-21T19:34:48.132ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "1119e3fe-6ba3-44eb-9b82-84847e54097f", "content-length": "2", "date": "Thu, 21 Nov 2024 19:34:47 GMT"} })
25432024-11-21T19:34:48.132ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
25442024-11-21T19:34:48.132ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "a4a3cb41-a654-4e88-81cb-ee1ef86a9388", "content-length": "51", "date": "Thu, 21 Nov 2024 19:34:47 GMT"} })
25452024-11-21T19:34:48.132ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
25462024-11-21T19:34:48.132ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
25472024-11-21T19:34:48.224ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "f3ac6906-e71f-4075-8607-62b5be5aec9c", "date": "Thu, 21 Nov 2024 19:34:48 GMT"} })
25482024-11-21T19:34:48.224ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
25492024-11-21T19:34:48.224ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
25502024-11-21T19:34:48.225ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "ceb0ebdc-0b8b-48b4-a08f-c6cb462b2142", "content-length": "682", "date": "Thu, 21 Nov 2024 19:34:47 GMT"} })
25512024-11-21T19:34:48.225ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
25522024-11-21T19:34:48.225ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
25532024-11-21T19:34:48.229ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
25542024-11-21T19:34:48.229ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
25552024-11-21T19:34:48.232ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
25562024-11-21T19:34:48.232ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
25572024-11-21T19:34:48.233ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
25582024-11-21T19:34:48.234ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
25592024-11-21T19:34:48.240ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 410.555326ms iteration = 4 status = {}
25602024-11-21T19:34:56.958ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 283 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = ac21a467-27e0-4bd7-a6e6-81f1888ef180 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
25612024-11-21T19:34:57.829ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 11 reason = Timeout
25622024-11-21T19:34:57.829ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
25632024-11-21T19:34:57.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
25642024-11-21T19:34:57.829ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 11 reason = Timeout
25652024-11-21T19:34:57.829ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
25662024-11-21T19:34:57.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
25672024-11-21T19:34:57.831ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
25682024-11-21T19:34:57.832ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
25692024-11-21T19:34:57.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
25702024-11-21T19:34:57.834ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 4.678762ms iteration = 11 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
25712024-11-21T19:34:57.834ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
25722024-11-21T19:34:57.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
25732024-11-21T19:34:57.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
25742024-11-21T19:34:57.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
25752024-11-21T19:34:57.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
25762024-11-21T19:34:57.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
25772024-11-21T19:34:57.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
25782024-11-21T19:34:57.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
25792024-11-21T19:34:57.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
25802024-11-21T19:34:57.851ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
25812024-11-21T19:34:57.851ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 22.143436ms iteration = 11 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
25822024-11-21T19:35:07.828ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 12 reason = Timeout
25832024-11-21T19:35:07.829ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
25842024-11-21T19:35:07.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
25852024-11-21T19:35:07.830ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 12 reason = Timeout
25862024-11-21T19:35:07.830ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
25872024-11-21T19:35:07.830ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
25882024-11-21T19:35:07.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
25892024-11-21T19:35:07.832ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
25902024-11-21T19:35:07.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
25912024-11-21T19:35:07.833ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
25922024-11-21T19:35:07.833ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
25932024-11-21T19:35:07.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
25942024-11-21T19:35:07.834ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 3.540318ms iteration = 12 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
25952024-11-21T19:35:07.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
25962024-11-21T19:35:07.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
25972024-11-21T19:35:07.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
25982024-11-21T19:35:07.838ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
25992024-11-21T19:35:07.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
26002024-11-21T19:35:07.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
26012024-11-21T19:35:07.849ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
26022024-11-21T19:35:07.849ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 20.218499ms iteration = 12 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
26032024-11-21T19:35:11.956ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 221 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 98008775-01eb-40fa-9c02-495d46bb6bcc response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
26042024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 5 reason = Timeout
26052024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
26062024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 13 reason = Timeout
26072024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 4 reason = Timeout
26082024-11-21T19:35:17.829ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:83 reason = blueprint disabled target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb
26092024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
26102024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 5 reason = Timeout
26112024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 19.47\u{b5}s iteration = 4 status = {"enabled":false,"target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb"}
26122024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 3 reason = Timeout
26132024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 5 reason = Timeout
26142024-11-21T19:35:17.829ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
26152024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
26162024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
26172024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 3 reason = Timeout
26182024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
26192024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
26202024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
26212024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 3 reason = Timeout
26222024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
26232024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
26242024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 3 reason = Timeout
26252024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 5 reason = Timeout
26262024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 5 reason = Timeout
26272024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 6 reason = Timeout
26282024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 5 reason = Timeout
26292024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
26302024-11-21T19:35:17.829ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 5 reason = Timeout
26312024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
26322024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 5 reason = Timeout
26332024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 3 reason = Timeout
26342024-11-21T19:35:17.830ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-11-21 19:25:17.829873642 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
26352024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
26362024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
26372024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 3 reason = Timeout
26382024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 3 reason = Timeout
26392024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
26402024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 5 reason = Timeout
26412024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
26422024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 13 reason = Timeout
26432024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
26442024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 3 reason = Timeout
26452024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
26462024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 5 reason = Timeout
26472024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
26482024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): activating background_task = instance_reincarnation iteration = 3 reason = Timeout
26492024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 5 reason = Timeout
26502024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
26512024-11-21T19:35:17.830ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
26522024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
26532024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 5 reason = Timeout
26542024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 5 reason = Timeout
26552024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
26562024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
26572024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 5 reason = Timeout
26582024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
26592024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
26602024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 3 reason = Timeout
26612024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
26622024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 5 reason = Timeout
26632024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
26642024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 5 reason = Timeout
26652024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
26662024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
26672024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
26682024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
26692024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
26702024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
26712024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
26722024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
26732024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
26742024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
26752024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
26762024-11-21T19:35:17.830ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0377cf96-baf3-4e09-a570-eb7c67037d4d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0d1b3b31-4591-4bcb-931d-e8e03365f0d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:1::1))) }], valid_until: Instant { tv_sec: 860, tv_nsec: 709641105 } })
26772024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
26782024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
26792024-11-21T19:35:17.830ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("4e669ff4-6dde-4cd2-8491-623460b56d8d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 860, tv_nsec: 709790317 } })
26802024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
26812024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
26822024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
26832024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
26842024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
26852024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
26862024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
26872024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
26882024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
26892024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
26902024-11-21T19:35:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
26912024-11-21T19:35:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
26922024-11-21T19:35:17.831ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
26932024-11-21T19:35:17.831ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 1.305187ms iteration = 3 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
26942024-11-21T19:35:17.831ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } dns_group = internal
26952024-11-21T19:35:17.831ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 1.5316ms iteration = 3 status = {"addresses":["[fd00:1122:3344:3::1]:5353","[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353"]}
26962024-11-21T19:35:17.832ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
26972024-11-21T19:35:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
26982024-11-21T19:35:17.833ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.68434ms iteration = 5 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
26992024-11-21T19:35:17.833ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27002024-11-21T19:35:17.833ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 860, tv_nsec: 713054161 } })
27012024-11-21T19:35:17.834ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
27022024-11-21T19:35:17.834ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27032024-11-21T19:35:17.834ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
27042024-11-21T19:35:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
27052024-11-21T19:35:17.835ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27062024-11-21T19:35:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
27072024-11-21T19:35:17.837ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
27082024-11-21T19:35:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
27092024-11-21T19:35:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
27102024-11-21T19:35:17.838ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
27112024-11-21T19:35:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
27122024-11-21T19:35:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
27132024-11-21T19:35:17.839ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
27142024-11-21T19:35:17.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
27152024-11-21T19:35:17.841ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27162024-11-21T19:35:17.842ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-11-21 19:25:17.829873642 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
27172024-11-21T19:35:17.842ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 12.743651ms iteration = 3 status = {"expiration":"2024-11-21T19:25:17.829873642Z","pruned":[]}
27182024-11-21T19:35:17.842ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27192024-11-21T19:35:17.843ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27202024-11-21T19:35:17.844ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
27212024-11-21T19:35:17.844ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
27222024-11-21T19:35:17.844ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
27232024-11-21T19:35:17.846ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
27242024-11-21T19:35:17.846ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
27252024-11-21T19:35:17.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
27262024-11-21T19:35:17.846ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 860, tv_nsec: 725572208 } })
27272024-11-21T19:35:17.846ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
27282024-11-21T19:35:17.846ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
27292024-11-21T19:35:17.846ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 16.44209ms iteration = 3 status = {"errors":[],"found_port_ok":[]}
27302024-11-21T19:35:17.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
27312024-11-21T19:35:17.846ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
27322024-11-21T19:35:17.848ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27332024-11-21T19:35:17.851ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
27342024-11-21T19:35:17.851ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
27352024-11-21T19:35:17.851ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
27362024-11-21T19:35:17.853ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27372024-11-21T19:35:17.855ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27382024-11-21T19:35:17.856ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27392024-11-21T19:35:17.857ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
27402024-11-21T19:35:17.857ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
27412024-11-21T19:35:17.857ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
27422024-11-21T19:35:17.858ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
27432024-11-21T19:35:17.859ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
27442024-11-21T19:35:17.859ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
27452024-11-21T19:35:17.859ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
27462024-11-21T19:35:17.859ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
27472024-11-21T19:35:17.860ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
27482024-11-21T19:35:17.860ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
27492024-11-21T19:35:17.861ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
27502024-11-21T19:35:17.861ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
27512024-11-21T19:35:17.861ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
27522024-11-21T19:35:17.861ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
27532024-11-21T19:35:17.861ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
27542024-11-21T19:35:17.862ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
27552024-11-21T19:35:17.862ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
27562024-11-21T19:35:17.862ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
27572024-11-21T19:35:17.862ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
27582024-11-21T19:35:17.862ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
27592024-11-21T19:35:17.863ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
27602024-11-21T19:35:17.864ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27612024-11-21T19:35:17.867ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
27622024-11-21T19:35:17.867ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
27632024-11-21T19:35:17.867ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
27642024-11-21T19:35:17.868ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
27652024-11-21T19:35:17.868ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27662024-11-21T19:35:17.868ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27672024-11-21T19:35:17.869ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
27682024-11-21T19:35:17.871ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
27692024-11-21T19:35:17.871ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
27702024-11-21T19:35:17.871ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 860, tv_nsec: 750773295 } })
27712024-11-21T19:35:17.871ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
27722024-11-21T19:35:17.872ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
27732024-11-21T19:35:17.872ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
27742024-11-21T19:35:17.872ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
27752024-11-21T19:35:17.872ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
27762024-11-21T19:35:17.872ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
27772024-11-21T19:35:17.872ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
27782024-11-21T19:35:17.878ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27792024-11-21T19:35:17.879ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27802024-11-21T19:35:17.879ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
27812024-11-21T19:35:17.882ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
27822024-11-21T19:35:17.882ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
27832024-11-21T19:35:17.883ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
27842024-11-21T19:35:17.884ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
27852024-11-21T19:35:17.884ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
27862024-11-21T19:35:17.884ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
27872024-11-21T19:35:17.884ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
27882024-11-21T19:35:17.884ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
27892024-11-21T19:35:17.884ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
27902024-11-21T19:35:17.884ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
27912024-11-21T19:35:17.884ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
27922024-11-21T19:35:17.889ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
27932024-11-21T19:35:17.889ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
27942024-11-21T19:35:17.898ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
27952024-11-21T19:35:17.898ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
27962024-11-21T19:35:17.898ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
27972024-11-21T19:35:17.898ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
27982024-11-21T19:35:17.901ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
27992024-11-21T19:35:17.901ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 71.408403ms iteration = 13 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
28002024-11-21T19:35:17.902ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0 total = 0
28012024-11-21T19:35:17.902ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:171 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0 vmms_found = 0
28022024-11-21T19:35:17.902ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 72.748632ms iteration = 3 status = {"errors":[],"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0,"vmms_found":0}
28032024-11-21T19:35:17.905ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
28042024-11-21T19:35:17.905ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
28052024-11-21T19:35:17.906ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 76.378021ms iteration = 5 status = {"errors":[],"records_set_to_done":[]}
28062024-11-21T19:35:17.908ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
28072024-11-21T19:35:17.908ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
28082024-11-21T19:35:17.908ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 78.741762ms iteration = 3 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
28092024-11-21T19:35:17.910ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 80.221882ms iteration = 5 status = {"errors":[],"garbage_collect_requested":[]}
28102024-11-21T19:35:17.911ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 81.426838ms iteration = 5 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
28112024-11-21T19:35:17.921ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
28122024-11-21T19:35:17.921ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 91.893838ms iteration = 5 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
28132024-11-21T19:35:17.928ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
28142024-11-21T19:35:17.928ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
28152024-11-21T19:35:17.932ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
28162024-11-21T19:35:17.932ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 0 version = 2
28172024-11-21T19:35:17.932ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
28182024-11-21T19:35:17.932ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
28192024-11-21T19:35:17.965ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
28202024-11-21T19:35:17.966ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
28212024-11-21T19:35:18.003ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
28222024-11-21T19:35:18.003ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
28232024-11-21T19:35:18.012ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
28242024-11-21T19:35:18.032ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
28252024-11-21T19:35:18.032ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
28262024-11-21T19:35:18.033ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "cac13f04-7c06-4826-8dfa-4b2c6b8c769d", "content-length": "26", "date": "Thu, 21 Nov 2024 19:35:17 GMT"} })
28272024-11-21T19:35:18.033ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
28282024-11-21T19:35:18.034ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
28292024-11-21T19:35:18.043ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
28302024-11-21T19:35:18.043ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
28312024-11-21T19:35:18.044ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "ad3b0da9-f211-4914-80ab-8173d7908271", "content-length": "372", "date": "Thu, 21 Nov 2024 19:35:17 GMT"} })
28322024-11-21T19:35:18.044ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
28332024-11-21T19:35:18.044ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
28342024-11-21T19:35:18.048ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
28352024-11-21T19:35:18.049ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
28362024-11-21T19:35:18.050ZWARNnexus (BackgroundTasks): instance reincarnation completed with errors background_task = instance_reincarnation file = nexus/src/app/background/tasks/instance_reincarnation.rs:103 instances_changed_state = 0 instances_found = 0 instances_reincarnated = 0 query_errors = 0 restart_errors = 0
28372024-11-21T19:35:18.050ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_reincarnation elapsed = 220.01711ms iteration = 3 status = {"changed_state":[],"disabled":false,"errors":[],"instances_found":{"Failed":0,"SagaUnwound":0},"instances_reincarnated":[],"restart_errors":[]}
28382024-11-21T19:35:18.052ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
28392024-11-21T19:35:18.052ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
28402024-11-21T19:35:18.075ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
28412024-11-21T19:35:18.081ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
28422024-11-21T19:35:18.081ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
28432024-11-21T19:35:18.081ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "fe16d3bb-5756-4279-bf0f-53eb582a92ae", "content-length": "26", "date": "Thu, 21 Nov 2024 19:35:17 GMT"} })
28442024-11-21T19:35:18.081ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
28452024-11-21T19:35:18.082ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
28462024-11-21T19:35:18.091ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
28472024-11-21T19:35:18.091ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
28482024-11-21T19:35:18.091ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
28492024-11-21T19:35:18.091ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
28502024-11-21T19:35:18.091ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 261.652996ms iteration = 13 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
28512024-11-21T19:35:18.091ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
28522024-11-21T19:35:18.091ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
28532024-11-21T19:35:18.091ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
28542024-11-21T19:35:18.092ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
28552024-11-21T19:35:18.094ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
28562024-11-21T19:35:18.095ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
28572024-11-21T19:35:18.095ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
28582024-11-21T19:35:18.097ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
28592024-11-21T19:35:18.097ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
28602024-11-21T19:35:18.097ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
28612024-11-21T19:35:18.101ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 270.96211ms iteration = 5 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
28622024-11-21T19:35:18.101ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 51 version = 1
28632024-11-21T19:35:18.101ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
28642024-11-21T19:35:18.101ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-11-21 19:33:31.797390 UTC dns_group = internal generation = 1
28652024-11-21T19:35:18.101ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 271.363166ms iteration = 3 status = {"generation":1}
28662024-11-21T19:35:18.102ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 272.574232ms iteration = 5 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
28672024-11-21T19:35:18.102ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
28682024-11-21T19:35:18.104ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
28692024-11-21T19:35:18.105ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
28702024-11-21T19:35:18.105ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
28712024-11-21T19:35:18.105ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
28722024-11-21T19:35:18.112ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
28732024-11-21T19:35:18.112ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
28742024-11-21T19:35:18.112ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
28752024-11-21T19:35:18.114ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 285.244401ms iteration = 5 status = 0
28762024-11-21T19:35:18.115ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 1 version = 2
28772024-11-21T19:35:18.115ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
28782024-11-21T19:35:18.115ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-11-21 19:33:32.758954 UTC dns_group = external generation = 2
28792024-11-21T19:35:18.115ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 285.361313ms iteration = 3 status = {"generation":2}
28802024-11-21T19:35:18.124ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
28812024-11-21T19:35:18.124ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
28822024-11-21T19:35:18.124ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "c01b6196-e9e9-4847-bb55-d59858b28db2", "date": "Thu, 21 Nov 2024 19:35:17 GMT"} })
28832024-11-21T19:35:18.124ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
28842024-11-21T19:35:18.125ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
28852024-11-21T19:35:18.127ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
28862024-11-21T19:35:18.127ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
28872024-11-21T19:35:18.127ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
28882024-11-21T19:35:18.130ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
28892024-11-21T19:35:18.130ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
28902024-11-21T19:35:18.131ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
28912024-11-21T19:35:18.131ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
28922024-11-21T19:35:18.131ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
28932024-11-21T19:35:18.134ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
28942024-11-21T19:35:18.134ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
28952024-11-21T19:35:18.134ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "7f6f44c7-b06f-4419-b73a-c484960a114a", "date": "Thu, 21 Nov 2024 19:35:17 GMT"} })
28962024-11-21T19:35:18.136ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 306.020289ms iteration = 5 status = {}
28972024-11-21T19:35:18.163ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
28982024-11-21T19:35:18.163ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
28992024-11-21T19:35:18.163ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
29002024-11-21T19:35:18.163ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
29012024-11-21T19:35:18.163ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 333.521366ms iteration = 5 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
29022024-11-21T19:35:18.163ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "1c96c839-f356-494a-be9a-40ee5a49b939", "content-length": "26", "date": "Thu, 21 Nov 2024 19:35:17 GMT"} })
29032024-11-21T19:35:18.163ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
29042024-11-21T19:35:18.165ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
29052024-11-21T19:35:18.169ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 339.289903ms iteration = 6 status = {"physical_disks_added":0}
29062024-11-21T19:35:18.169ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
29072024-11-21T19:35:18.169ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
29082024-11-21T19:35:18.171ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
29092024-11-21T19:35:18.174ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
29102024-11-21T19:35:18.174ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
29112024-11-21T19:35:18.174ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
29122024-11-21T19:35:18.174ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 345.315034ms iteration = 5 status = {}
29132024-11-21T19:35:18.175ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
29142024-11-21T19:35:18.178ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
29152024-11-21T19:35:18.178ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
29162024-11-21T19:35:18.178ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
29172024-11-21T19:35:18.179ZINFOnexus (BackgroundTasks): found Silo/DNS/TLS config (changed) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-11-21T19:33:15.446964Z, time_modified: 2024-11-21T19:33:15.446964Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }) } file = nexus/src/app/background/tasks/external_endpoints.rs:94
29182024-11-21T19:35:18.179ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 349.55841ms iteration = 3 status = {"by_dns_name":{"default-silo.sys.oxide.test":{"silo_id":"001de000-5110-4000-8000-000000000000","tls_certs":[]},"recovery.sys.oxide.test":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]}},"default_endpoint":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]},"warnings":["silo 001de000-5110-4000-8000-000000000000 with DNS name \\"default-silo.sys.oxide.test\\" has no usable certificates"]}
29192024-11-21T19:35:18.200ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
29202024-11-21T19:35:18.200ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::8]:5353/config", status: 204, headers: {"x-request-id": "bd2f92c2-791b-4f17-a5b3-60cea85988e5", "date": "Thu, 21 Nov 2024 19:35:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
29212024-11-21T19:35:18.200ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
29222024-11-21T19:35:18.202ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::9]:5353/config", status: 204, headers: {"x-request-id": "75312df4-a2a8-43a9-b629-25960e96bee5", "date": "Thu, 21 Nov 2024 19:35:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
29232024-11-21T19:35:18.202ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
29242024-11-21T19:35:18.203ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
29252024-11-21T19:35:18.203ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 373.564012ms iteration = 5 status = {"generation":2,"server_results":{"[fd00:1122:3344:101::8]:5353":{"Ok":null},"[fd00:1122:3344:101::9]:5353":{"Ok":null}}}
29262024-11-21T19:35:18.236ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
29272024-11-21T19:35:18.253ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
29282024-11-21T19:35:18.253ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "8da23546-0862-49bb-afc9-1116df71186b", "content-length": "1", "date": "Thu, 21 Nov 2024 19:35:18 GMT"} })
29292024-11-21T19:35:18.255ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
29302024-11-21T19:35:18.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
29312024-11-21T19:35:18.258ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 428.292903ms iteration = 5 status = null
29322024-11-21T19:35:18.331ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
29332024-11-21T19:35:18.332ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:3::1]:5353/config", status: 204, headers: {"x-request-id": "93f6033f-4e00-4e80-b6bc-dccacafe4569", "date": "Thu, 21 Nov 2024 19:35:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
29342024-11-21T19:35:18.332ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
29352024-11-21T19:35:18.333ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:1::1]:5353/config", status: 204, headers: {"x-request-id": "cb4a8219-8e8c-41e9-9a05-f9acc2c10ed4", "date": "Thu, 21 Nov 2024 19:35:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
29362024-11-21T19:35:18.333ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
29372024-11-21T19:35:18.335ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:2::1]:5353/config", status: 204, headers: {"x-request-id": "b06f634e-c79f-4c62-a268-fd3481f7df01", "date": "Thu, 21 Nov 2024 19:35:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
29382024-11-21T19:35:18.335ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
29392024-11-21T19:35:18.336ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
29402024-11-21T19:35:18.336ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 507.063625ms iteration = 5 status = {"generation":1,"server_results":{"[fd00:1122:3344:1::1]:5353":{"Ok":null},"[fd00:1122:3344:2::1]:5353":{"Ok":null},"[fd00:1122:3344:3::1]:5353":{"Ok":null}}}
29412024-11-21T19:35:18.338ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
29422024-11-21T19:35:18.339ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "181bceab-b6d1-4f4a-9500-da812af6a055", "content-length": "691", "date": "Thu, 21 Nov 2024 19:35:17 GMT"} })
29432024-11-21T19:35:18.339ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
29442024-11-21T19:35:18.339ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
29452024-11-21T19:35:18.339ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
29462024-11-21T19:35:18.341ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 511.806769ms iteration = 5 status = {}
29472024-11-21T19:35:18.357ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
29482024-11-21T19:35:18.358ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
29492024-11-21T19:35:18.360ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
29502024-11-21T19:35:18.360ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
29512024-11-21T19:35:18.360ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
29522024-11-21T19:35:18.368ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
29532024-11-21T19:35:18.369ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "0ec04c5f-6fcf-4dae-ae5e-225e40795271", "content-length": "134", "date": "Thu, 21 Nov 2024 19:35:18 GMT"} })
29542024-11-21T19:35:18.369ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
29552024-11-21T19:35:18.369ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
29562024-11-21T19:35:18.369ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
29572024-11-21T19:35:18.369ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
29582024-11-21T19:35:18.369ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
29592024-11-21T19:35:18.369ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
29602024-11-21T19:35:18.369ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
29612024-11-21T19:35:18.369ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
29622024-11-21T19:35:18.369ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
29632024-11-21T19:35:18.369ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
29642024-11-21T19:35:18.369ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
29652024-11-21T19:35:18.369ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
29662024-11-21T19:35:18.370ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
29672024-11-21T19:35:18.372ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
29682024-11-21T19:35:18.372ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "f935e1f2-8e3c-45e9-8d35-4b727aac3e3d", "content-length": "2", "date": "Thu, 21 Nov 2024 19:35:18 GMT"} })
29692024-11-21T19:35:18.372ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
29702024-11-21T19:35:18.372ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "d915e862-083e-480b-bc3d-6520df499ea9", "content-length": "51", "date": "Thu, 21 Nov 2024 19:35:18 GMT"} })
29712024-11-21T19:35:18.372ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
29722024-11-21T19:35:18.372ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
29732024-11-21T19:35:18.511ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "2ca825a2-8fb7-42b5-83bb-f7ca1fa88abb", "date": "Thu, 21 Nov 2024 19:35:17 GMT"} })
29742024-11-21T19:35:18.511ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
29752024-11-21T19:35:18.511ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
29762024-11-21T19:35:18.511ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "11572398-9b9f-4189-9154-fd23c279313a", "content-length": "682", "date": "Thu, 21 Nov 2024 19:35:18 GMT"} })
29772024-11-21T19:35:18.511ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
29782024-11-21T19:35:18.511ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
29792024-11-21T19:35:18.515ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
29802024-11-21T19:35:18.515ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
29812024-11-21T19:35:18.518ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
29822024-11-21T19:35:18.518ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
29832024-11-21T19:35:18.520ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
29842024-11-21T19:35:18.520ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
29852024-11-21T19:35:18.526ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 696.929022ms iteration = 5 status = {}
29862024-11-21T19:35:26.959ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 253 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 68622946-bcc1-44df-ba6e-cb03691f7df0 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
29872024-11-21T19:35:27.829ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 14 reason = Timeout
29882024-11-21T19:35:27.829ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
29892024-11-21T19:35:27.829ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
29902024-11-21T19:35:27.830ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 14 reason = Timeout
29912024-11-21T19:35:27.830ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
29922024-11-21T19:35:27.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
29932024-11-21T19:35:27.831ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
29942024-11-21T19:35:27.832ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
29952024-11-21T19:35:27.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
29962024-11-21T19:35:27.834ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
29972024-11-21T19:35:27.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
29982024-11-21T19:35:27.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
29992024-11-21T19:35:27.834ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 4.253567ms iteration = 14 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
30002024-11-21T19:35:27.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
30012024-11-21T19:35:27.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
30022024-11-21T19:35:27.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30032024-11-21T19:35:27.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
30042024-11-21T19:35:27.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
30052024-11-21T19:35:27.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
30062024-11-21T19:35:27.852ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
30072024-11-21T19:35:27.852ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 22.921446ms iteration = 14 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
30082024-11-21T19:35:37.830ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 15 reason = Timeout
30092024-11-21T19:35:37.830ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
30102024-11-21T19:35:37.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
30112024-11-21T19:35:37.830ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 15 reason = Timeout
30122024-11-21T19:35:37.830ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
30132024-11-21T19:35:37.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
30142024-11-21T19:35:37.831ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30152024-11-21T19:35:37.832ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
30162024-11-21T19:35:37.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
30172024-11-21T19:35:37.834ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 3.73702ms iteration = 15 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
30182024-11-21T19:35:37.834ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
30192024-11-21T19:35:37.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
30202024-11-21T19:35:37.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
30212024-11-21T19:35:37.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
30222024-11-21T19:35:37.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
30232024-11-21T19:35:37.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30242024-11-21T19:35:37.841ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
30252024-11-21T19:35:37.841ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
30262024-11-21T19:35:37.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
30272024-11-21T19:35:37.851ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
30282024-11-21T19:35:37.851ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 21.895263ms iteration = 15 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
30292024-11-21T19:35:41.958ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 255 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = cfbbe21f-25a2-4005-9584-24efd9d6234f response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
30302024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 6 reason = Timeout
30312024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
30322024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 16 reason = Timeout
30332024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 7 reason = Timeout
30342024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
30352024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
30362024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 6 reason = Timeout
30372024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 6 reason = Timeout
30382024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
30392024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
30402024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 6 reason = Timeout
30412024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 6 reason = Timeout
30422024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 6 reason = Timeout
30432024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
30442024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
30452024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 6 reason = Timeout
30462024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
30472024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
30482024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
30492024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
30502024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
30512024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
30522024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
30532024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
30542024-11-21T19:35:47.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
30552024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 16 reason = Timeout
30562024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 6 reason = Timeout
30572024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 6 reason = Timeout
30582024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
30592024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
30602024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
30612024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 6 reason = Timeout
30622024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 6 reason = Timeout
30632024-11-21T19:35:47.831ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
30642024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
30652024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
30662024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 6 reason = Timeout
30672024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 6 reason = Timeout
30682024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
30692024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
30702024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 6 reason = Timeout
30712024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
30722024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
30732024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
30742024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
30752024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
30762024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
30772024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
30782024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
30792024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
30802024-11-21T19:35:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
30812024-11-21T19:35:47.832ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30822024-11-21T19:35:47.832ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 890, tv_nsec: 711686242 } })
30832024-11-21T19:35:47.832ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 2.867439ms iteration = 6 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
30842024-11-21T19:35:47.833ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
30852024-11-21T19:35:47.833ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30862024-11-21T19:35:47.834ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 890, tv_nsec: 712736266 } })
30872024-11-21T19:35:47.834ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
30882024-11-21T19:35:47.834ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30892024-11-21T19:35:47.836ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
30902024-11-21T19:35:47.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
30912024-11-21T19:35:47.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
30922024-11-21T19:35:47.837ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
30932024-11-21T19:35:47.837ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
30942024-11-21T19:35:47.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
30952024-11-21T19:35:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
30962024-11-21T19:35:47.838ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
30972024-11-21T19:35:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
30982024-11-21T19:35:47.839ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
30992024-11-21T19:35:47.839ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
31002024-11-21T19:35:47.839ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
31012024-11-21T19:35:47.839ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
31022024-11-21T19:35:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
31032024-11-21T19:35:47.840ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 9.425886ms iteration = 6 status = {"errors":[],"garbage_collect_requested":[]}
31042024-11-21T19:35:47.841ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
31052024-11-21T19:35:47.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
31062024-11-21T19:35:47.842ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
31072024-11-21T19:35:47.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
31082024-11-21T19:35:47.843ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
31092024-11-21T19:35:47.843ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
31102024-11-21T19:35:47.843ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
31112024-11-21T19:35:47.845ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
31122024-11-21T19:35:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
31132024-11-21T19:35:47.846ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
31142024-11-21T19:35:47.846ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
31152024-11-21T19:35:47.846ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
31162024-11-21T19:35:47.846ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
31172024-11-21T19:35:47.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
31182024-11-21T19:35:47.846ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
31192024-11-21T19:35:47.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
31202024-11-21T19:35:47.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
31212024-11-21T19:35:47.848ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
31222024-11-21T19:35:47.850ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
31232024-11-21T19:35:47.850ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 19.578282ms iteration = 6 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
31242024-11-21T19:35:47.851ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
31252024-11-21T19:35:47.851ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
31262024-11-21T19:35:47.851ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
31272024-11-21T19:35:47.877ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
31282024-11-21T19:35:47.877ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
31292024-11-21T19:35:47.877ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
31302024-11-21T19:35:47.878ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
31312024-11-21T19:35:47.879ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
31322024-11-21T19:35:47.879ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 48.124483ms iteration = 6 status = {"errors":[],"records_set_to_done":[]}
31332024-11-21T19:35:47.881ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
31342024-11-21T19:35:47.881ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
31352024-11-21T19:35:47.881ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 50.273032ms iteration = 6 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
31362024-11-21T19:35:47.881ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 50.448824ms iteration = 6 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
31372024-11-21T19:35:47.881ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
31382024-11-21T19:35:47.881ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
31392024-11-21T19:35:47.881ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
31402024-11-21T19:35:47.883ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
31412024-11-21T19:35:47.883ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
31422024-11-21T19:35:47.884ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
31432024-11-21T19:35:47.884ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
31442024-11-21T19:35:47.884ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 890, tv_nsec: 763201341 } })
31452024-11-21T19:35:47.884ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
31462024-11-21T19:35:47.884ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
31472024-11-21T19:35:47.884ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
31482024-11-21T19:35:47.885ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
31492024-11-21T19:35:47.885ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
31502024-11-21T19:35:47.885ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
31512024-11-21T19:35:47.885ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
31522024-11-21T19:35:47.886ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
31532024-11-21T19:35:47.886ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
31542024-11-21T19:35:47.888ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
31552024-11-21T19:35:47.888ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
31562024-11-21T19:35:47.888ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
31572024-11-21T19:35:47.888ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
31582024-11-21T19:35:47.888ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
31592024-11-21T19:35:47.888ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
31602024-11-21T19:35:47.891ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 60.017452ms iteration = 16 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
31612024-11-21T19:35:47.895ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
31622024-11-21T19:35:47.898ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
31632024-11-21T19:35:47.898ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
31642024-11-21T19:35:47.898ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
31652024-11-21T19:35:47.901ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 70.958818ms iteration = 6 status = 0
31662024-11-21T19:35:47.910ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
31672024-11-21T19:35:47.910ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
31682024-11-21T19:35:47.910ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
31692024-11-21T19:35:47.910ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 80.401245ms iteration = 16 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
31702024-11-21T19:35:47.910ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "f65d6301-107f-474c-b3cf-9c0662e2cf5c", "content-length": "26", "date": "Thu, 21 Nov 2024 19:35:47 GMT"} })
31712024-11-21T19:35:47.910ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
31722024-11-21T19:35:47.912ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
31732024-11-21T19:35:47.917ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 87.001382ms iteration = 6 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
31742024-11-21T19:35:47.927ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 96.607541ms iteration = 6 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
31752024-11-21T19:35:47.940ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
31762024-11-21T19:35:47.940ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
31772024-11-21T19:35:47.941ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "04172abb-e647-4614-9750-9024cb21715d", "content-length": "26", "date": "Thu, 21 Nov 2024 19:35:47 GMT"} })
31782024-11-21T19:35:47.941ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
31792024-11-21T19:35:47.942ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
31802024-11-21T19:35:47.958ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 128.458296ms iteration = 7 status = {"physical_disks_added":0}
31812024-11-21T19:35:47.966ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
31822024-11-21T19:35:47.967ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
31832024-11-21T19:35:48.014ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
31842024-11-21T19:35:48.014ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
31852024-11-21T19:35:48.014ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "99684b69-bf6f-4d23-96c8-682a65405b75", "content-length": "372", "date": "Thu, 21 Nov 2024 19:35:47 GMT"} })
31862024-11-21T19:35:48.014ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
31872024-11-21T19:35:48.015ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
31882024-11-21T19:35:48.019ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
31892024-11-21T19:35:48.019ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
31902024-11-21T19:35:48.019ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "3e6ce741-dc2f-4abd-92ec-f6579e05cb8e", "content-length": "26", "date": "Thu, 21 Nov 2024 19:35:47 GMT"} })
31912024-11-21T19:35:48.019ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
31922024-11-21T19:35:48.021ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
31932024-11-21T19:35:48.021ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
31942024-11-21T19:35:48.022ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "72ed406e-84e8-410c-86cc-7f1d616b3821", "content-length": "1", "date": "Thu, 21 Nov 2024 19:35:48 GMT"} })
31952024-11-21T19:35:48.023ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
31962024-11-21T19:35:48.023ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
31972024-11-21T19:35:48.025ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 195.603834ms iteration = 6 status = null
31982024-11-21T19:35:48.029ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
31992024-11-21T19:35:48.029ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
32002024-11-21T19:35:48.030ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "1f25bdf9-b3d4-4dff-b409-840ebea85fb4", "date": "Thu, 21 Nov 2024 19:35:47 GMT"} })
32012024-11-21T19:35:48.030ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
32022024-11-21T19:35:48.030ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
32032024-11-21T19:35:48.032ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
32042024-11-21T19:35:48.032ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
32052024-11-21T19:35:48.035ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
32062024-11-21T19:35:48.035ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
32072024-11-21T19:35:48.035ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 205.936812ms iteration = 6 status = {}
32082024-11-21T19:35:48.038ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
32092024-11-21T19:35:48.038ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
32102024-11-21T19:35:48.042ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "6f069631-b05f-4170-9430-8bb8c370e034", "date": "Thu, 21 Nov 2024 19:35:47 GMT"} })
32112024-11-21T19:35:48.043ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 212.5495ms iteration = 6 status = {}
32122024-11-21T19:35:48.107ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
32132024-11-21T19:35:48.108ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
32142024-11-21T19:35:48.109ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
32152024-11-21T19:35:48.111ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
32162024-11-21T19:35:48.111ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
32172024-11-21T19:35:48.112ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
32182024-11-21T19:35:48.113ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
32192024-11-21T19:35:48.115ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
32202024-11-21T19:35:48.115ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
32212024-11-21T19:35:48.115ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
32222024-11-21T19:35:48.126ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
32232024-11-21T19:35:48.126ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
32242024-11-21T19:35:48.128ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
32252024-11-21T19:35:48.128ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
32262024-11-21T19:35:48.129ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
32272024-11-21T19:35:48.138ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
32282024-11-21T19:35:48.138ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "75b1f958-59b8-497d-81cc-125a46bff06c", "content-length": "134", "date": "Thu, 21 Nov 2024 19:35:48 GMT"} })
32292024-11-21T19:35:48.138ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
32302024-11-21T19:35:48.138ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
32312024-11-21T19:35:48.139ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
32322024-11-21T19:35:48.139ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
32332024-11-21T19:35:48.139ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
32342024-11-21T19:35:48.139ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
32352024-11-21T19:35:48.139ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
32362024-11-21T19:35:48.139ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
32372024-11-21T19:35:48.139ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
32382024-11-21T19:35:48.139ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
32392024-11-21T19:35:48.139ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
32402024-11-21T19:35:48.139ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
32412024-11-21T19:35:48.139ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
32422024-11-21T19:35:48.141ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
32432024-11-21T19:35:48.141ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "5a5f0ef7-6b7d-4829-8a90-ac9815d8d799", "content-length": "2", "date": "Thu, 21 Nov 2024 19:35:48 GMT"} })
32442024-11-21T19:35:48.141ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
32452024-11-21T19:35:48.141ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "f49d4f96-c8a7-4f0d-b45d-49f58fdef4ab", "content-length": "51", "date": "Thu, 21 Nov 2024 19:35:48 GMT"} })
32462024-11-21T19:35:48.141ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
32472024-11-21T19:35:48.141ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
32482024-11-21T19:35:48.153ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
32492024-11-21T19:35:48.236ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "eaa30900-fd65-4780-86ad-857dece1d442", "date": "Thu, 21 Nov 2024 19:35:47 GMT"} })
32502024-11-21T19:35:48.236ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
32512024-11-21T19:35:48.236ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
32522024-11-21T19:35:48.236ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "19c07d1e-3792-40ec-9298-6e9dd44b7b3c", "content-length": "691", "date": "Thu, 21 Nov 2024 19:35:47 GMT"} })
32532024-11-21T19:35:48.236ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
32542024-11-21T19:35:48.236ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
32552024-11-21T19:35:48.236ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
32562024-11-21T19:35:48.236ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "0529c9ca-7c06-40d8-9c65-0bfa37dbfe6e", "content-length": "682", "date": "Thu, 21 Nov 2024 19:35:47 GMT"} })
32572024-11-21T19:35:48.236ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
32582024-11-21T19:35:48.237ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
32592024-11-21T19:35:48.237ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 407.578906ms iteration = 6 status = {}
32602024-11-21T19:35:48.240ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
32612024-11-21T19:35:48.240ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
32622024-11-21T19:35:48.243ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
32632024-11-21T19:35:48.243ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
32642024-11-21T19:35:48.245ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
32652024-11-21T19:35:48.245ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
32662024-11-21T19:35:48.252ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 422.548666ms iteration = 6 status = {}
32672024-11-21T19:35:56.957ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 418 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 5b2bd6f9-c194-4ba9-9045-acd19278a5b5 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
32682024-11-21T19:35:57.830ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 17 reason = Timeout
32692024-11-21T19:35:57.830ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
32702024-11-21T19:35:57.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
32712024-11-21T19:35:57.830ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 17 reason = Timeout
32722024-11-21T19:35:57.830ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
32732024-11-21T19:35:57.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
32742024-11-21T19:35:57.832ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
32752024-11-21T19:35:57.835ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
32762024-11-21T19:35:57.835ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
32772024-11-21T19:35:57.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
32782024-11-21T19:35:57.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
32792024-11-21T19:35:57.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
32802024-11-21T19:35:57.841ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
32812024-11-21T19:35:57.843ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
32822024-11-21T19:35:57.843ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
32832024-11-21T19:35:57.844ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
32842024-11-21T19:35:57.854ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
32852024-11-21T19:35:57.854ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 24.309365ms iteration = 17 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
32862024-11-21T19:35:57.867ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
32872024-11-21T19:35:57.867ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
32882024-11-21T19:35:57.901ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 70.912848ms iteration = 17 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
32892024-11-21T19:36:07.830ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 18 reason = Timeout
32902024-11-21T19:36:07.830ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
32912024-11-21T19:36:07.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
32922024-11-21T19:36:07.831ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 18 reason = Timeout
32932024-11-21T19:36:07.831ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
32942024-11-21T19:36:07.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
32952024-11-21T19:36:07.832ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
32962024-11-21T19:36:07.834ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
32972024-11-21T19:36:07.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
32982024-11-21T19:36:07.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
32992024-11-21T19:36:07.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
33002024-11-21T19:36:07.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
33012024-11-21T19:36:07.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
33022024-11-21T19:36:07.841ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
33032024-11-21T19:36:07.841ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
33042024-11-21T19:36:07.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
33052024-11-21T19:36:07.851ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
33062024-11-21T19:36:07.852ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 21.754681ms iteration = 18 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
33072024-11-21T19:36:07.859ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
33082024-11-21T19:36:07.860ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
33092024-11-21T19:36:07.861ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 30.447996ms iteration = 18 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
33102024-11-21T19:36:11.959ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 284 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 7ad8bc7f-2810-4961-8d8c-c22c20cfbeed response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
33112024-11-21T19:36:16.951ZINFOnexus (dropshot_internal): accepted connection file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023 local_addr = [fd00:1122:3344:101::c]:12221 remote_addr = [fd00:1122:3344:101::d]:45872
33122024-11-21T19:36:16.961ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:45872 req_id = b499d07d-1fb6-4c3d-a4df-31ed9ba282d5 roles = RoleSet { roles: {} } uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
33132024-11-21T19:36:16.961ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:45872 req_id = b499d07d-1fb6-4c3d-a4df-31ed9ba282d5 resource = Database result = Ok(()) uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
33142024-11-21T19:36:16.969ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 17762 local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:45872 req_id = b499d07d-1fb6-4c3d-a4df-31ed9ba282d5 response_code = 200 uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
33152024-11-21T19:36:16.973ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:45872 req_id = 16a98ea0-5fd5-41ab-8df1-b18eb699a413 roles = RoleSet { roles: {} } uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
33162024-11-21T19:36:16.973ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:45872 req_id = 16a98ea0-5fd5-41ab-8df1-b18eb699a413 resource = Database result = Ok(()) uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
33172024-11-21T19:36:16.978ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 9719 local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:45872 req_id = 16a98ea0-5fd5-41ab-8df1-b18eb699a413 response_code = 200 uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
33182024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 7 reason = Timeout
33192024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
33202024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 8 reason = Timeout
33212024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 6 reason = Timeout
33222024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
33232024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 7 reason = Timeout
33242024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 19 reason = Timeout
33252024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
33262024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
33272024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 4 reason = Timeout
33282024-11-21T19:36:17.830ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
33292024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 7 reason = Timeout
33302024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
33312024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
33322024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 4 reason = Timeout
33332024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
33342024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
33352024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 7 reason = Timeout
33362024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 4 reason = Timeout
33372024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 4 reason = Timeout
33382024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = crdb_node_id_collector iteration = 3 reason = Timeout
33392024-11-21T19:36:17.830ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-11-21 19:26:17.830363937 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
33402024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
33412024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 4 reason = Timeout
33422024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
33432024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
33442024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
33452024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 5 reason = Timeout
33462024-11-21T19:36:17.830ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:83 reason = blueprint disabled target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb
33472024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 17.621\u{b5}s iteration = 5 status = {"enabled":false,"target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb"}
33482024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 7 reason = Timeout
33492024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 4 reason = Timeout
33502024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
33512024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
33522024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 7 reason = Timeout
33532024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
33542024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 7 reason = Timeout
33552024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
33562024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 6 reason = Timeout
33572024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 4 reason = Timeout
33582024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
33592024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
33602024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
33612024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
33622024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
33632024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
33642024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
33652024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
33662024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
33672024-11-21T19:36:17.830ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
33682024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 19 reason = Timeout
33692024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): activating background_task = instance_reincarnation iteration = 4 reason = Timeout
33702024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
33712024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
33722024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 4 reason = Timeout
33732024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
33742024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 7 reason = Timeout
33752024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 7 reason = Timeout
33762024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
33772024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 7 reason = Timeout
33782024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
33792024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
33802024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
33812024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
33822024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
33832024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 7 reason = Timeout
33842024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 7 reason = Timeout
33852024-11-21T19:36:17.831ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
33862024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
33872024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
33882024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 7 reason = Timeout
33892024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
33902024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
33912024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
33922024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 7 reason = Timeout
33932024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
33942024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
33952024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 4 reason = Timeout
33962024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
33972024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
33982024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
33992024-11-21T19:36:17.831ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("4e669ff4-6dde-4cd2-8491-623460b56d8d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 920, tv_nsec: 709509854 } })
34002024-11-21T19:36:17.831ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0377cf96-baf3-4e09-a570-eb7c67037d4d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0d1b3b31-4591-4bcb-931d-e8e03365f0d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:1::1))) }], valid_until: Instant { tv_sec: 920, tv_nsec: 709510203 } })
34012024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
34022024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
34032024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
34042024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
34052024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
34062024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
34072024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
34082024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
34092024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
34102024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
34112024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
34122024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
34132024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
34142024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
34152024-11-21T19:36:17.831ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
34162024-11-21T19:36:17.832ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
34172024-11-21T19:36:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
34182024-11-21T19:36:17.832ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } dns_group = internal
34192024-11-21T19:36:17.832ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 1.5342ms iteration = 4 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
34202024-11-21T19:36:17.832ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 1.725132ms iteration = 4 status = {"addresses":["[fd00:1122:3344:3::1]:5353","[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353"]}
34212024-11-21T19:36:17.835ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
34222024-11-21T19:36:17.835ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
34232024-11-21T19:36:17.838ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
34242024-11-21T19:36:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
34252024-11-21T19:36:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
34262024-11-21T19:36:17.839ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
34272024-11-21T19:36:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
34282024-11-21T19:36:17.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
34292024-11-21T19:36:17.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
34302024-11-21T19:36:17.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
34312024-11-21T19:36:17.842ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 920, tv_nsec: 720484000 } })
34322024-11-21T19:36:17.842ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
34332024-11-21T19:36:17.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
34342024-11-21T19:36:17.842ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
34352024-11-21T19:36:17.843ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
34362024-11-21T19:36:17.843ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-11-21 19:26:17.830363937 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
34372024-11-21T19:36:17.843ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 13.341108ms iteration = 4 status = {"expiration":"2024-11-21T19:26:17.830363937Z","pruned":[]}
34382024-11-21T19:36:17.844ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 14.095668ms iteration = 7 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
34392024-11-21T19:36:17.844ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
34402024-11-21T19:36:17.845ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
34412024-11-21T19:36:17.845ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
34422024-11-21T19:36:17.848ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
34432024-11-21T19:36:17.848ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
34442024-11-21T19:36:17.848ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
34452024-11-21T19:36:17.849ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
34462024-11-21T19:36:17.849ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
34472024-11-21T19:36:17.849ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
34482024-11-21T19:36:17.849ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
34492024-11-21T19:36:17.849ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
34502024-11-21T19:36:17.849ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
34512024-11-21T19:36:17.849ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
34522024-11-21T19:36:17.850ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
34532024-11-21T19:36:17.850ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
34542024-11-21T19:36:17.853ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 920, tv_nsec: 731244964 } })
34552024-11-21T19:36:17.853ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
34562024-11-21T19:36:17.853ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
34572024-11-21T19:36:17.853ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
34582024-11-21T19:36:17.854ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
34592024-11-21T19:36:17.857ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
34602024-11-21T19:36:17.857ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
34612024-11-21T19:36:17.857ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
34622024-11-21T19:36:17.875ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
34632024-11-21T19:36:17.875ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 44.102539ms iteration = 4 status = {"errors":[],"found_port_ok":[]}
34642024-11-21T19:36:17.879ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
34652024-11-21T19:36:17.889ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
34662024-11-21T19:36:17.889ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
34672024-11-21T19:36:17.889ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
34682024-11-21T19:36:17.889ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
34692024-11-21T19:36:17.892ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
34702024-11-21T19:36:17.893ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
34712024-11-21T19:36:17.906ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
34722024-11-21T19:36:17.909ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
34732024-11-21T19:36:17.909ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
34742024-11-21T19:36:17.909ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
34752024-11-21T19:36:17.923ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
34762024-11-21T19:36:17.924ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
34772024-11-21T19:36:17.926ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 95.592587ms iteration = 7 status = {"errors":[],"records_set_to_done":[]}
34782024-11-21T19:36:17.928ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
34792024-11-21T19:36:17.931ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
34802024-11-21T19:36:17.931ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
34812024-11-21T19:36:17.931ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
34822024-11-21T19:36:17.931ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 920, tv_nsec: 809959996 } })
34832024-11-21T19:36:17.932ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
34842024-11-21T19:36:17.934ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
34852024-11-21T19:36:17.934ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
34862024-11-21T19:36:17.934ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
34872024-11-21T19:36:17.934ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
34882024-11-21T19:36:17.934ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
34892024-11-21T19:36:17.935ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
34902024-11-21T19:36:17.935ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
34912024-11-21T19:36:17.935ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0 total = 0
34922024-11-21T19:36:17.935ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:171 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0 vmms_found = 0
34932024-11-21T19:36:17.935ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 104.857892ms iteration = 4 status = {"errors":[],"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0,"vmms_found":0}
34942024-11-21T19:36:17.936ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
34952024-11-21T19:36:17.936ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 105.893495ms iteration = 7 status = {"errors":[],"garbage_collect_requested":[]}
34962024-11-21T19:36:17.938ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
34972024-11-21T19:36:17.938ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
34982024-11-21T19:36:17.938ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
34992024-11-21T19:36:17.938ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
35002024-11-21T19:36:17.939ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
35012024-11-21T19:36:17.940ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
35022024-11-21T19:36:17.940ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
35032024-11-21T19:36:17.940ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
35042024-11-21T19:36:17.941ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
35052024-11-21T19:36:17.942ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
35062024-11-21T19:36:17.944ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
35072024-11-21T19:36:17.944ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
35082024-11-21T19:36:17.944ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
35092024-11-21T19:36:17.944ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
35102024-11-21T19:36:17.945ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
35112024-11-21T19:36:17.947ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
35122024-11-21T19:36:17.947ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
35132024-11-21T19:36:17.948ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
35142024-11-21T19:36:17.949ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
35152024-11-21T19:36:17.949ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
35162024-11-21T19:36:17.949ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
35172024-11-21T19:36:17.949ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
35182024-11-21T19:36:17.949ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
35192024-11-21T19:36:17.952ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
35202024-11-21T19:36:17.952ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
35212024-11-21T19:36:17.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
35222024-11-21T19:36:17.954ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
35232024-11-21T19:36:17.954ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
35242024-11-21T19:36:17.954ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
35252024-11-21T19:36:17.954ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
35262024-11-21T19:36:17.955ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
35272024-11-21T19:36:17.955ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
35282024-11-21T19:36:17.955ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
35292024-11-21T19:36:17.955ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
35302024-11-21T19:36:17.958ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
35312024-11-21T19:36:17.958ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
35322024-11-21T19:36:17.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
35332024-11-21T19:36:17.958ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
35342024-11-21T19:36:17.958ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
35352024-11-21T19:36:17.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
35362024-11-21T19:36:17.960ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
35372024-11-21T19:36:17.960ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
35382024-11-21T19:36:17.960ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
35392024-11-21T19:36:17.968ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
35402024-11-21T19:36:17.968ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
35412024-11-21T19:36:17.968ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
35422024-11-21T19:36:17.968ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
35432024-11-21T19:36:17.968ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
35442024-11-21T19:36:17.970ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
35452024-11-21T19:36:17.972ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 142.122039ms iteration = 7 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
35462024-11-21T19:36:17.973ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
35472024-11-21T19:36:17.973ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
35482024-11-21T19:36:17.973ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
35492024-11-21T19:36:17.977ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
35502024-11-21T19:36:17.981ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
35512024-11-21T19:36:17.981ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
35522024-11-21T19:36:17.981ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
35532024-11-21T19:36:17.982ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
35542024-11-21T19:36:17.982ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
35552024-11-21T19:36:17.987ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
35562024-11-21T19:36:17.989ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
35572024-11-21T19:36:17.990ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
35582024-11-21T19:36:17.990ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 159.754155ms iteration = 4 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
35592024-11-21T19:36:17.990ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
35602024-11-21T19:36:17.990ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
35612024-11-21T19:36:17.990ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
35622024-11-21T19:36:17.990ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 160.610786ms iteration = 19 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
35632024-11-21T19:36:17.991ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
35642024-11-21T19:36:17.993ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
35652024-11-21T19:36:17.994ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
35662024-11-21T19:36:17.994ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
35672024-11-21T19:36:17.995ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
35682024-11-21T19:36:17.995ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
35692024-11-21T19:36:17.996ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
35702024-11-21T19:36:17.996ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
35712024-11-21T19:36:17.996ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
35722024-11-21T19:36:17.999ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
35732024-11-21T19:36:17.999ZWARNnexus (BackgroundTasks): instance reincarnation completed with errors background_task = instance_reincarnation file = nexus/src/app/background/tasks/instance_reincarnation.rs:103 instances_changed_state = 0 instances_found = 0 instances_reincarnated = 0 query_errors = 0 restart_errors = 0
35742024-11-21T19:36:17.999ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_reincarnation elapsed = 168.38939ms iteration = 4 status = {"changed_state":[],"disabled":false,"errors":[],"instances_found":{"Failed":0,"SagaUnwound":0},"instances_reincarnated":[],"restart_errors":[]}
35752024-11-21T19:36:17.999ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
35762024-11-21T19:36:18.000ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
35772024-11-21T19:36:18.001ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
35782024-11-21T19:36:18.004ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
35792024-11-21T19:36:18.004ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
35802024-11-21T19:36:18.004ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
35812024-11-21T19:36:18.005ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
35822024-11-21T19:36:18.005ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
35832024-11-21T19:36:18.005ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 174.305469ms iteration = 19 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
35842024-11-21T19:36:18.005ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
35852024-11-21T19:36:18.006ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 175.599106ms iteration = 7 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
35862024-11-21T19:36:18.011ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 181.483065ms iteration = 7 status = 0
35872024-11-21T19:36:18.013ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
35882024-11-21T19:36:18.013ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 182.220985ms iteration = 7 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
35892024-11-21T19:36:18.019ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
35902024-11-21T19:36:18.019ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
35912024-11-21T19:36:18.020ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "aa039a84-d39b-433f-a3ee-46bada20484c", "content-length": "26", "date": "Thu, 21 Nov 2024 19:36:17 GMT"} })
35922024-11-21T19:36:18.020ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
35932024-11-21T19:36:18.021ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
35942024-11-21T19:36:18.022ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
35952024-11-21T19:36:18.023ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
35962024-11-21T19:36:18.024ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
35972024-11-21T19:36:18.024ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
35982024-11-21T19:36:18.025ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
35992024-11-21T19:36:18.026ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
36002024-11-21T19:36:18.028ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
36012024-11-21T19:36:18.028ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
36022024-11-21T19:36:18.028ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
36032024-11-21T19:36:18.035ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
36042024-11-21T19:36:18.035ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
36052024-11-21T19:36:18.035ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
36062024-11-21T19:36:18.035ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
36072024-11-21T19:36:18.036ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
36082024-11-21T19:36:18.036ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
36092024-11-21T19:36:18.036ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 205.336593ms iteration = 7 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
36102024-11-21T19:36:18.037ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
36112024-11-21T19:36:18.038ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
36122024-11-21T19:36:18.038ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 0 version = 2
36132024-11-21T19:36:18.038ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
36142024-11-21T19:36:18.038ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
36152024-11-21T19:36:18.040ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
36162024-11-21T19:36:18.041ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
36172024-11-21T19:36:18.041ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
36182024-11-21T19:36:18.042ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
36192024-11-21T19:36:18.043ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
36202024-11-21T19:36:18.043ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
36212024-11-21T19:36:18.043ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
36222024-11-21T19:36:18.044ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 51 version = 1
36232024-11-21T19:36:18.044ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
36242024-11-21T19:36:18.044ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-11-21 19:33:31.797390 UTC dns_group = internal generation = 1
36252024-11-21T19:36:18.044ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 214.594017ms iteration = 4 status = {"generation":1}
36262024-11-21T19:36:18.045ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 1 version = 2
36272024-11-21T19:36:18.045ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
36282024-11-21T19:36:18.045ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-11-21 19:33:32.758954 UTC dns_group = external generation = 2
36292024-11-21T19:36:18.045ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 215.510719ms iteration = 4 status = {"generation":2}
36302024-11-21T19:36:18.048ZDEBGnexus (BackgroundTasks): activation complete background_task = crdb_node_id_collector elapsed = 218.017573ms iteration = 3 status = {"nsuccess":5}
36312024-11-21T19:36:18.073ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 242.304707ms iteration = 7 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
36322024-11-21T19:36:18.083ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-11-21T19:33:15.446964Z, time_modified: 2024-11-21T19:33:15.446964Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }) }
36332024-11-21T19:36:18.083ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 252.96703ms iteration = 4 status = {"by_dns_name":{"default-silo.sys.oxide.test":{"silo_id":"001de000-5110-4000-8000-000000000000","tls_certs":[]},"recovery.sys.oxide.test":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]}},"default_endpoint":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]},"warnings":["silo 001de000-5110-4000-8000-000000000000 with DNS name \\"default-silo.sys.oxide.test\\" has no usable certificates"]}
36342024-11-21T19:36:18.091ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
36352024-11-21T19:36:18.120ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
36362024-11-21T19:36:18.120ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
36372024-11-21T19:36:18.120ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "4c60b6c9-c5f8-4da4-a742-85d553331a70", "content-length": "26", "date": "Thu, 21 Nov 2024 19:36:17 GMT"} })
36382024-11-21T19:36:18.120ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
36392024-11-21T19:36:18.122ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
36402024-11-21T19:36:18.131ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 301.029752ms iteration = 8 status = {"physical_disks_added":0}
36412024-11-21T19:36:18.135ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
36422024-11-21T19:36:18.135ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
36432024-11-21T19:36:18.135ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "be545b92-f3ac-47e4-bb70-a74e816c5057", "content-length": "26", "date": "Thu, 21 Nov 2024 19:36:17 GMT"} })
36442024-11-21T19:36:18.135ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
36452024-11-21T19:36:18.137ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
36462024-11-21T19:36:18.195ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 364.618141ms iteration = 7 status = {}
36472024-11-21T19:36:18.203ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
36482024-11-21T19:36:18.203ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::8]:5353/config", status: 204, headers: {"x-request-id": "daa0596a-9360-45e8-8a77-178459036fcb", "date": "Thu, 21 Nov 2024 19:36:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
36492024-11-21T19:36:18.203ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
36502024-11-21T19:36:18.205ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::9]:5353/config", status: 204, headers: {"x-request-id": "c7fb6c44-9a38-43de-acd1-0bbc3eefdb78", "date": "Thu, 21 Nov 2024 19:36:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
36512024-11-21T19:36:18.205ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
36522024-11-21T19:36:18.206ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
36532024-11-21T19:36:18.206ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 376.641293ms iteration = 6 status = {"generation":2,"server_results":{"[fd00:1122:3344:101::8]:5353":{"Ok":null},"[fd00:1122:3344:101::9]:5353":{"Ok":null}}}
36542024-11-21T19:36:18.265ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
36552024-11-21T19:36:18.266ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "38d8ddcd-d954-4dce-859f-c689a91c0752", "content-length": "691", "date": "Thu, 21 Nov 2024 19:36:18 GMT"} })
36562024-11-21T19:36:18.266ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
36572024-11-21T19:36:18.266ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
36582024-11-21T19:36:18.266ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
36592024-11-21T19:36:18.268ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 438.226186ms iteration = 7 status = {}
36602024-11-21T19:36:18.270ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
36612024-11-21T19:36:18.270ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
36622024-11-21T19:36:18.271ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "c434e2ab-38b3-43a5-9549-a0faf1cb5979", "content-length": "372", "date": "Thu, 21 Nov 2024 19:36:18 GMT"} })
36632024-11-21T19:36:18.271ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
36642024-11-21T19:36:18.271ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
36652024-11-21T19:36:18.277ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
36662024-11-21T19:36:18.278ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
36672024-11-21T19:36:18.278ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "cd9296cc-a9b0-4562-9918-e0975490285f", "content-length": "1", "date": "Thu, 21 Nov 2024 19:36:18 GMT"} })
36682024-11-21T19:36:18.280ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
36692024-11-21T19:36:18.280ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
36702024-11-21T19:36:18.282ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 451.811708ms iteration = 7 status = null
36712024-11-21T19:36:18.285ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
36722024-11-21T19:36:18.285ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
36732024-11-21T19:36:18.286ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "681b5d38-3be4-4466-ba83-89c8118cb7ae", "date": "Thu, 21 Nov 2024 19:36:18 GMT"} })
36742024-11-21T19:36:18.286ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
36752024-11-21T19:36:18.286ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
36762024-11-21T19:36:18.287ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
36772024-11-21T19:36:18.287ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
36782024-11-21T19:36:18.291ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
36792024-11-21T19:36:18.291ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
36802024-11-21T19:36:18.294ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
36812024-11-21T19:36:18.294ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
36822024-11-21T19:36:18.294ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "1e58373d-66e5-4349-842c-422102e1eb10", "date": "Thu, 21 Nov 2024 19:36:18 GMT"} })
36832024-11-21T19:36:18.296ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 465.201816ms iteration = 7 status = {}
36842024-11-21T19:36:18.337ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
36852024-11-21T19:36:18.338ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:1::1]:5353/config", status: 204, headers: {"x-request-id": "773dccd0-b632-4a19-9c5b-125ffe1351a5", "date": "Thu, 21 Nov 2024 19:36:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
36862024-11-21T19:36:18.338ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
36872024-11-21T19:36:18.339ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:3::1]:5353/config", status: 204, headers: {"x-request-id": "8fd9c30d-1721-409b-ab83-dadefb14bb70", "date": "Thu, 21 Nov 2024 19:36:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
36882024-11-21T19:36:18.339ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
36892024-11-21T19:36:18.340ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:2::1]:5353/config", status: 204, headers: {"x-request-id": "dfcaf755-9bfa-489d-8e5b-3ae2a41f8715", "date": "Thu, 21 Nov 2024 19:36:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
36902024-11-21T19:36:18.340ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
36912024-11-21T19:36:18.342ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
36922024-11-21T19:36:18.342ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 512.163963ms iteration = 6 status = {"generation":1,"server_results":{"[fd00:1122:3344:1::1]:5353":{"Ok":null},"[fd00:1122:3344:2::1]:5353":{"Ok":null},"[fd00:1122:3344:3::1]:5353":{"Ok":null}}}
36932024-11-21T19:36:18.367ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
36942024-11-21T19:36:18.367ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
36952024-11-21T19:36:18.369ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
36962024-11-21T19:36:18.369ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
36972024-11-21T19:36:18.369ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
36982024-11-21T19:36:18.475ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
36992024-11-21T19:36:18.475ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "1315f613-6356-4534-81b8-3e9ba0541633", "content-length": "134", "date": "Thu, 21 Nov 2024 19:36:18 GMT"} })
37002024-11-21T19:36:18.475ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
37012024-11-21T19:36:18.475ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
37022024-11-21T19:36:18.476ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
37032024-11-21T19:36:18.476ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
37042024-11-21T19:36:18.476ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
37052024-11-21T19:36:18.476ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
37062024-11-21T19:36:18.476ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
37072024-11-21T19:36:18.476ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
37082024-11-21T19:36:18.476ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
37092024-11-21T19:36:18.476ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
37102024-11-21T19:36:18.476ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
37112024-11-21T19:36:18.476ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
37122024-11-21T19:36:18.476ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
37132024-11-21T19:36:18.478ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
37142024-11-21T19:36:18.478ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "165ad08b-504e-47d0-b8b2-c52546684d5f", "content-length": "2", "date": "Thu, 21 Nov 2024 19:36:18 GMT"} })
37152024-11-21T19:36:18.478ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
37162024-11-21T19:36:18.478ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "c55d9e4b-64fb-4e0d-a396-45716678f86c", "content-length": "51", "date": "Thu, 21 Nov 2024 19:36:18 GMT"} })
37172024-11-21T19:36:18.478ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
37182024-11-21T19:36:18.478ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
37192024-11-21T19:36:18.591ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "cd983b72-692b-49dc-bb3b-0807e3a3f6e2", "date": "Thu, 21 Nov 2024 19:36:18 GMT"} })
37202024-11-21T19:36:18.591ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
37212024-11-21T19:36:18.591ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
37222024-11-21T19:36:18.591ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "fe7f9004-44bf-4898-811e-7e90a806186f", "content-length": "682", "date": "Thu, 21 Nov 2024 19:36:18 GMT"} })
37232024-11-21T19:36:18.591ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
37242024-11-21T19:36:18.592ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
37252024-11-21T19:36:18.595ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
37262024-11-21T19:36:18.595ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
37272024-11-21T19:36:18.597ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
37282024-11-21T19:36:18.597ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
37292024-11-21T19:36:18.599ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
37302024-11-21T19:36:18.599ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
37312024-11-21T19:36:18.605ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 775.338719ms iteration = 7 status = {}
37322024-11-21T19:36:26.980ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 622 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 8fdddf16-c931-4542-9f12-fe547c171134 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
37332024-11-21T19:36:27.830ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 20 reason = Timeout
37342024-11-21T19:36:27.830ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
37352024-11-21T19:36:27.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
37362024-11-21T19:36:27.831ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 20 reason = Timeout
37372024-11-21T19:36:27.831ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
37382024-11-21T19:36:27.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
37392024-11-21T19:36:27.833ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
37402024-11-21T19:36:27.833ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
37412024-11-21T19:36:27.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
37422024-11-21T19:36:27.835ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 3.902622ms iteration = 20 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
37432024-11-21T19:36:27.835ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
37442024-11-21T19:36:27.835ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
37452024-11-21T19:36:27.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
37462024-11-21T19:36:27.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
37472024-11-21T19:36:27.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
37482024-11-21T19:36:27.839ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
37492024-11-21T19:36:27.841ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
37502024-11-21T19:36:27.841ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
37512024-11-21T19:36:27.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
37522024-11-21T19:36:27.850ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
37532024-11-21T19:36:27.850ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 19.525001ms iteration = 20 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
37542024-11-21T19:36:37.830ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 21 reason = Timeout
37552024-11-21T19:36:37.830ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
37562024-11-21T19:36:37.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
37572024-11-21T19:36:37.831ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 21 reason = Timeout
37582024-11-21T19:36:37.831ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
37592024-11-21T19:36:37.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
37602024-11-21T19:36:37.833ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
37612024-11-21T19:36:37.833ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
37622024-11-21T19:36:37.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
37632024-11-21T19:36:37.834ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 3.566057ms iteration = 21 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
37642024-11-21T19:36:37.835ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
37652024-11-21T19:36:37.835ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
37662024-11-21T19:36:37.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
37672024-11-21T19:36:37.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
37682024-11-21T19:36:37.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
37692024-11-21T19:36:37.839ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
37702024-11-21T19:36:37.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
37712024-11-21T19:36:37.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
37722024-11-21T19:36:37.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
37732024-11-21T19:36:37.851ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
37742024-11-21T19:36:37.851ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 20.144259ms iteration = 21 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
37752024-11-21T19:36:41.959ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 274 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 84d4b898-6d6c-4a87-a6cc-b7f4e501d64d response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
37762024-11-21T19:36:47.830ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 8 reason = Timeout
37772024-11-21T19:36:47.830ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
37782024-11-21T19:36:47.830ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 22 reason = Timeout
37792024-11-21T19:36:47.830ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 8 reason = Timeout
37802024-11-21T19:36:47.830ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
37812024-11-21T19:36:47.830ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 8 reason = Timeout
37822024-11-21T19:36:47.830ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
37832024-11-21T19:36:47.830ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
37842024-11-21T19:36:47.830ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 8 reason = Timeout
37852024-11-21T19:36:47.830ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
37862024-11-21T19:36:47.830ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 8 reason = Timeout
37872024-11-21T19:36:47.830ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 9 reason = Timeout
37882024-11-21T19:36:47.830ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
37892024-11-21T19:36:47.830ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 8 reason = Timeout
37902024-11-21T19:36:47.830ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
37912024-11-21T19:36:47.830ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 8 reason = Timeout
37922024-11-21T19:36:47.831ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
37932024-11-21T19:36:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
37942024-11-21T19:36:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
37952024-11-21T19:36:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
37962024-11-21T19:36:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
37972024-11-21T19:36:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
37982024-11-21T19:36:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
37992024-11-21T19:36:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
38002024-11-21T19:36:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
38012024-11-21T19:36:47.831ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 22 reason = Timeout
38022024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 8 reason = Timeout
38032024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
38042024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
38052024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 8 reason = Timeout
38062024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
38072024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
38082024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 8 reason = Timeout
38092024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
38102024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 8 reason = Timeout
38112024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
38122024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 8 reason = Timeout
38132024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
38142024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 8 reason = Timeout
38152024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
38162024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 8 reason = Timeout
38172024-11-21T19:36:47.832ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
38182024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
38192024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
38202024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
38212024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
38222024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
38232024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
38242024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
38252024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
38262024-11-21T19:36:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
38272024-11-21T19:36:47.833ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 2.715496ms iteration = 8 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
38282024-11-21T19:36:47.833ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 950, tv_nsec: 711494650 } })
38292024-11-21T19:36:47.833ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
38302024-11-21T19:36:47.834ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 950, tv_nsec: 712169179 } })
38312024-11-21T19:36:47.834ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
38322024-11-21T19:36:47.835ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
38332024-11-21T19:36:47.835ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
38342024-11-21T19:36:47.835ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
38352024-11-21T19:36:47.836ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
38362024-11-21T19:36:47.836ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
38372024-11-21T19:36:47.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
38382024-11-21T19:36:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
38392024-11-21T19:36:47.839ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
38402024-11-21T19:36:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
38412024-11-21T19:36:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
38422024-11-21T19:36:47.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
38432024-11-21T19:36:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
38442024-11-21T19:36:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
38452024-11-21T19:36:47.840ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
38462024-11-21T19:36:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
38472024-11-21T19:36:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
38482024-11-21T19:36:47.840ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
38492024-11-21T19:36:47.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
38502024-11-21T19:36:47.843ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
38512024-11-21T19:36:47.844ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
38522024-11-21T19:36:47.844ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 12.933583ms iteration = 8 status = {"errors":[],"records_set_to_done":[]}
38532024-11-21T19:36:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
38542024-11-21T19:36:47.845ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
38552024-11-21T19:36:47.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
38562024-11-21T19:36:47.847ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
38572024-11-21T19:36:47.847ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
38582024-11-21T19:36:47.847ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
38592024-11-21T19:36:47.849ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
38602024-11-21T19:36:47.849ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
38612024-11-21T19:36:47.850ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
38622024-11-21T19:36:47.850ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
38632024-11-21T19:36:47.850ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
38642024-11-21T19:36:47.850ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
38652024-11-21T19:36:47.851ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 19.402959ms iteration = 22 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
38662024-11-21T19:36:47.851ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 950, tv_nsec: 729302628 } })
38672024-11-21T19:36:47.851ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
38682024-11-21T19:36:47.852ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
38692024-11-21T19:36:47.853ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
38702024-11-21T19:36:47.857ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
38712024-11-21T19:36:47.857ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
38722024-11-21T19:36:47.857ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
38732024-11-21T19:36:47.858ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
38742024-11-21T19:36:47.858ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
38752024-11-21T19:36:47.858ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
38762024-11-21T19:36:47.876ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
38772024-11-21T19:36:47.878ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
38782024-11-21T19:36:47.878ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
38792024-11-21T19:36:47.878ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
38802024-11-21T19:36:47.879ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
38812024-11-21T19:36:47.879ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
38822024-11-21T19:36:47.882ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
38832024-11-21T19:36:47.882ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 51.311826ms iteration = 22 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
38842024-11-21T19:36:47.883ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 52.186658ms iteration = 8 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
38852024-11-21T19:36:47.883ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
38862024-11-21T19:36:47.883ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
38872024-11-21T19:36:47.883ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 51.672701ms iteration = 8 status = {"errors":[],"garbage_collect_requested":[]}
38882024-11-21T19:36:47.885ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
38892024-11-21T19:36:47.885ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
38902024-11-21T19:36:47.885ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
38912024-11-21T19:36:47.885ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
38922024-11-21T19:36:47.885ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
38932024-11-21T19:36:47.885ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
38942024-11-21T19:36:47.885ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
38952024-11-21T19:36:47.885ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
38962024-11-21T19:36:47.888ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
38972024-11-21T19:36:47.888ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
38982024-11-21T19:36:47.888ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
38992024-11-21T19:36:47.888ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
39002024-11-21T19:36:47.889ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
39012024-11-21T19:36:47.891ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
39022024-11-21T19:36:47.891ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
39032024-11-21T19:36:47.892ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
39042024-11-21T19:36:47.893ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 62.707247ms iteration = 8 status = 0
39052024-11-21T19:36:47.900ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 69.430958ms iteration = 9 status = {"physical_disks_added":0}
39062024-11-21T19:36:47.900ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
39072024-11-21T19:36:47.900ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 68.501015ms iteration = 8 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
39082024-11-21T19:36:47.915ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 83.759579ms iteration = 8 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
39092024-11-21T19:36:47.923ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
39102024-11-21T19:36:47.924ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
39112024-11-21T19:36:47.926ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 94.381392ms iteration = 8 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
39122024-11-21T19:36:47.926ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
39132024-11-21T19:36:47.928ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
39142024-11-21T19:36:47.928ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
39152024-11-21T19:36:47.929ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
39162024-11-21T19:36:47.931ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
39172024-11-21T19:36:47.931ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
39182024-11-21T19:36:47.931ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "c586b099-0cb5-4eae-bd89-dd5a41b46024", "content-length": "26", "date": "Thu, 21 Nov 2024 19:36:47 GMT"} })
39192024-11-21T19:36:47.931ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
39202024-11-21T19:36:47.932ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
39212024-11-21T19:36:47.932ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
39222024-11-21T19:36:47.932ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 100.186248ms iteration = 8 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
39232024-11-21T19:36:47.933ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
39242024-11-21T19:36:47.964ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
39252024-11-21T19:36:47.964ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
39262024-11-21T19:36:47.966ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
39272024-11-21T19:36:47.968ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
39282024-11-21T19:36:47.968ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
39292024-11-21T19:36:47.968ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
39302024-11-21T19:36:47.969ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
39312024-11-21T19:36:47.971ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
39322024-11-21T19:36:47.971ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
39332024-11-21T19:36:47.971ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
39342024-11-21T19:36:47.999ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
39352024-11-21T19:36:47.999ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
39362024-11-21T19:36:47.999ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "46a3c154-2bfe-46b5-8d8c-7eaa5b4b8bc3", "content-length": "26", "date": "Thu, 21 Nov 2024 19:36:47 GMT"} })
39372024-11-21T19:36:47.999ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
39382024-11-21T19:36:48.001ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
39392024-11-21T19:36:48.035ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
39402024-11-21T19:36:48.035ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
39412024-11-21T19:36:48.039ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
39422024-11-21T19:36:48.039ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
39432024-11-21T19:36:48.039ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "94b3b9bd-28e1-4099-a932-3c876db76935", "content-length": "26", "date": "Thu, 21 Nov 2024 19:36:47 GMT"} })
39442024-11-21T19:36:48.039ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
39452024-11-21T19:36:48.039ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 208.947372ms iteration = 8 status = {}
39462024-11-21T19:36:48.040ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
39472024-11-21T19:36:48.076ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
39482024-11-21T19:36:48.076ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "d5a14416-391c-4b85-8878-edfbb31fc0f9", "content-length": "1", "date": "Thu, 21 Nov 2024 19:36:47 GMT"} })
39492024-11-21T19:36:48.079ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
39502024-11-21T19:36:48.079ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
39512024-11-21T19:36:48.081ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 250.427536ms iteration = 8 status = null
39522024-11-21T19:36:48.083ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "f13bf7a8-6d95-4162-b7ed-8782ea4de0a8", "content-length": "372", "date": "Thu, 21 Nov 2024 19:36:48 GMT"} })
39532024-11-21T19:36:48.083ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
39542024-11-21T19:36:48.083ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
39552024-11-21T19:36:48.098ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
39562024-11-21T19:36:48.098ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
39572024-11-21T19:36:48.098ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "e7253111-4bf4-42f1-a1fc-4bc566fb5138", "date": "Thu, 21 Nov 2024 19:36:48 GMT"} })
39582024-11-21T19:36:48.098ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
39592024-11-21T19:36:48.098ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
39602024-11-21T19:36:48.100ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
39612024-11-21T19:36:48.101ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
39622024-11-21T19:36:48.161ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
39632024-11-21T19:36:48.162ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "7b8ad81d-eace-4c59-9ca7-eaecde6d22e4", "content-length": "691", "date": "Thu, 21 Nov 2024 19:36:48 GMT"} })
39642024-11-21T19:36:48.162ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
39652024-11-21T19:36:48.162ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
39662024-11-21T19:36:48.162ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
39672024-11-21T19:36:48.164ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 333.0931ms iteration = 8 status = {}
39682024-11-21T19:36:48.165ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
39692024-11-21T19:36:48.166ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
39702024-11-21T19:36:48.169ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
39712024-11-21T19:36:48.169ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
39722024-11-21T19:36:48.170ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "c59edc51-c775-4247-ae06-88747ee77262", "date": "Thu, 21 Nov 2024 19:36:48 GMT"} })
39732024-11-21T19:36:48.171ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 339.405675ms iteration = 8 status = {}
39742024-11-21T19:36:48.182ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
39752024-11-21T19:36:48.183ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
39762024-11-21T19:36:48.184ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
39772024-11-21T19:36:48.184ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
39782024-11-21T19:36:48.185ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
39792024-11-21T19:36:48.195ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
39802024-11-21T19:36:48.195ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "8e4152de-0f1a-42a9-9112-87bea4c9fc03", "content-length": "134", "date": "Thu, 21 Nov 2024 19:36:47 GMT"} })
39812024-11-21T19:36:48.195ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
39822024-11-21T19:36:48.195ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
39832024-11-21T19:36:48.196ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
39842024-11-21T19:36:48.196ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
39852024-11-21T19:36:48.196ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
39862024-11-21T19:36:48.196ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
39872024-11-21T19:36:48.196ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
39882024-11-21T19:36:48.196ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
39892024-11-21T19:36:48.196ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
39902024-11-21T19:36:48.196ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
39912024-11-21T19:36:48.196ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
39922024-11-21T19:36:48.196ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
39932024-11-21T19:36:48.196ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
39942024-11-21T19:36:48.198ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
39952024-11-21T19:36:48.198ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "e957ca33-4935-45d6-95ba-1371e68eda16", "content-length": "2", "date": "Thu, 21 Nov 2024 19:36:47 GMT"} })
39962024-11-21T19:36:48.198ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
39972024-11-21T19:36:48.198ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "ff44b2bc-3241-4f2d-bfda-e30a8a153a1b", "content-length": "51", "date": "Thu, 21 Nov 2024 19:36:47 GMT"} })
39982024-11-21T19:36:48.198ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
39992024-11-21T19:36:48.198ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
40002024-11-21T19:36:48.290ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "e4653ae2-6e5a-4595-b917-19a9cdf8da5d", "date": "Thu, 21 Nov 2024 19:36:48 GMT"} })
40012024-11-21T19:36:48.290ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
40022024-11-21T19:36:48.290ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
40032024-11-21T19:36:48.290ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "df7e3ef3-3338-4e0d-b0fe-752a727fe9f3", "content-length": "682", "date": "Thu, 21 Nov 2024 19:36:48 GMT"} })
40042024-11-21T19:36:48.290ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
40052024-11-21T19:36:48.291ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
40062024-11-21T19:36:48.293ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
40072024-11-21T19:36:48.293ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
40082024-11-21T19:36:48.297ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
40092024-11-21T19:36:48.297ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
40102024-11-21T19:36:48.299ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
40112024-11-21T19:36:48.300ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
40122024-11-21T19:36:48.306ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 475.92292ms iteration = 8 status = {}
40132024-11-21T19:36:48.364ZDEBGnexus (producer-registration-task): registering / renewing oximeter producer lease with Nexus
40142024-11-21T19:36:48.364ZDEBGnexus (producer-registration-task): using nexus address for registration addr = [fd00:1122:3344:101::c]:12221
40152024-11-21T19:36:48.395ZDEBGnexus (producer-registration-task): client request body = Some(Body) method = POST uri = http://[fd00:1122:3344:101::c]:12221/metrics/producers
40162024-11-21T19:36:48.395ZINFOnexus (dropshot_internal): accepted connection file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023 local_addr = [fd00:1122:3344:101::c]:12221 remote_addr = [fd00:1122:3344:101::c]:46325
40172024-11-21T19:36:48.395ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:46325 req_id = 68c02cc6-7221-4c43-a32b-a23d6b6af22c roles = RoleSet { roles: {} } uri = /metrics/producers
40182024-11-21T19:36:48.395ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:46325 req_id = 68c02cc6-7221-4c43-a32b-a23d6b6af22c resource = Database result = Ok(()) uri = /metrics/producers
40192024-11-21T19:36:48.431ZINFOnexus: assigned collector to new producer collector_id = 5fb99117-c685-44d5-bec2-c58adb63b619 file = nexus/src/app/oximeter.rs:76 producer_id = 915e11b8-d7cb-4730-9906-7980540813dc
40202024-11-21T19:36:48.432ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 36525 local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:46325 req_id = 68c02cc6-7221-4c43-a32b-a23d6b6af22c response_code = 201 uri = /metrics/producers
40212024-11-21T19:36:48.432ZDEBGnexus (producer-registration-task): client response result = Ok(Response { url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", status: 201, headers: {"content-type": "application/json", "x-request-id": "68c02cc6-7221-4c43-a32b-a23d6b6af22c", "content-length": "41", "date": "Thu, 21 Nov 2024 19:36:48 GMT"} })
40222024-11-21T19:36:48.433ZDEBGnexus (producer-registration-task): registered with nexus successfully
40232024-11-21T19:36:48.433ZDEBGnexus (producer-registration-task): pausing until time to renew lease lease_duration = 600s wait_period = 150s
40242024-11-21T19:36:56.959ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 457 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 7da31ecd-c840-4856-8212-107607cf42c7 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
40252024-11-21T19:36:57.830ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 23 reason = Timeout
40262024-11-21T19:36:57.831ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
40272024-11-21T19:36:57.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
40282024-11-21T19:36:57.831ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 23 reason = Timeout
40292024-11-21T19:36:57.831ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
40302024-11-21T19:36:57.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
40312024-11-21T19:36:57.833ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
40322024-11-21T19:36:57.833ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
40332024-11-21T19:36:57.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
40342024-11-21T19:36:57.835ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 3.913942ms iteration = 23 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
40352024-11-21T19:36:57.835ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
40362024-11-21T19:36:57.835ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
40372024-11-21T19:36:57.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
40382024-11-21T19:36:57.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
40392024-11-21T19:36:57.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
40402024-11-21T19:36:57.839ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
40412024-11-21T19:36:57.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
40422024-11-21T19:36:57.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
40432024-11-21T19:36:57.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
40442024-11-21T19:36:57.853ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
40452024-11-21T19:36:57.853ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 22.990858ms iteration = 23 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
40462024-11-21T19:37:07.831ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 24 reason = Timeout
40472024-11-21T19:37:07.831ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
40482024-11-21T19:37:07.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
40492024-11-21T19:37:07.831ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 24 reason = Timeout
40502024-11-21T19:37:07.831ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
40512024-11-21T19:37:07.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
40522024-11-21T19:37:07.833ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
40532024-11-21T19:37:07.834ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
40542024-11-21T19:37:07.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
40552024-11-21T19:37:07.836ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 4.312108ms iteration = 24 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
40562024-11-21T19:37:07.836ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
40572024-11-21T19:37:07.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
40582024-11-21T19:37:07.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
40592024-11-21T19:37:07.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
40602024-11-21T19:37:07.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
40612024-11-21T19:37:07.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
40622024-11-21T19:37:07.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
40632024-11-21T19:37:07.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
40642024-11-21T19:37:07.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
40652024-11-21T19:37:07.854ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
40662024-11-21T19:37:07.854ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 23.369412ms iteration = 24 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
40672024-11-21T19:37:11.959ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 299 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 2ae49d77-7a87-47ad-858f-a42ba681ecc6 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
40682024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 9 reason = Timeout
40692024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
40702024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 25 reason = Timeout
40712024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
40722024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 6 reason = Timeout
40732024-11-21T19:37:17.831ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:83 reason = blueprint disabled target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb
40742024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 5 reason = Timeout
40752024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
40762024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 19.25\u{b5}s iteration = 6 status = {"enabled":false,"target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb"}
40772024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
40782024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 5 reason = Timeout
40792024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 5 reason = Timeout
40802024-11-21T19:37:17.831ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
40812024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 10 reason = Timeout
40822024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
40832024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
40842024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 9 reason = Timeout
40852024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 9 reason = Timeout
40862024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 9 reason = Timeout
40872024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
40882024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
40892024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 5 reason = Timeout
40902024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
40912024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 9 reason = Timeout
40922024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
40932024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 7 reason = Timeout
40942024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
40952024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 7 reason = Timeout
40962024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 5 reason = Timeout
40972024-11-21T19:37:17.831ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-11-21 19:27:17.831436462 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
40982024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 9 reason = Timeout
40992024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
41002024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
41012024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 5 reason = Timeout
41022024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
41032024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 5 reason = Timeout
41042024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 9 reason = Timeout
41052024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
41062024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
41072024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
41082024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
41092024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
41102024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
41112024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
41122024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 9 reason = Timeout
41132024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
41142024-11-21T19:37:17.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
41152024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
41162024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 9 reason = Timeout
41172024-11-21T19:37:17.832ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
41182024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 9 reason = Timeout
41192024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
41202024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
41212024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 9 reason = Timeout
41222024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
41232024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
41242024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 25 reason = Timeout
41252024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
41262024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 9 reason = Timeout
41272024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
41282024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
41292024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
41302024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
41312024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 5 reason = Timeout
41322024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): activating background_task = instance_reincarnation iteration = 5 reason = Timeout
41332024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 9 reason = Timeout
41342024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
41352024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
41362024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
41372024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
41382024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
41392024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 9 reason = Timeout
41402024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 5 reason = Timeout
41412024-11-21T19:37:17.832ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0377cf96-baf3-4e09-a570-eb7c67037d4d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0d1b3b31-4591-4bcb-931d-e8e03365f0d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:1::1))) }], valid_until: Instant { tv_sec: 980, tv_nsec: 709424622 } })
41422024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
41432024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
41442024-11-21T19:37:17.832ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("4e669ff4-6dde-4cd2-8491-623460b56d8d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 980, tv_nsec: 709493533 } })
41452024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
41462024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
41472024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
41482024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
41492024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
41502024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
41512024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
41522024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
41532024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
41542024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
41552024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
41562024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
41572024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 1.063324ms iteration = 5 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
41582024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } dns_group = internal
41592024-11-21T19:37:17.832ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 1.398498ms iteration = 5 status = {"addresses":["[fd00:1122:3344:3::1]:5353","[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353"]}
41602024-11-21T19:37:17.833ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
41612024-11-21T19:37:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
41622024-11-21T19:37:17.836ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 980, tv_nsec: 713658279 } })
41632024-11-21T19:37:17.836ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
41642024-11-21T19:37:17.837ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 980, tv_nsec: 715121178 } })
41652024-11-21T19:37:17.838ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
41662024-11-21T19:37:17.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
41672024-11-21T19:37:17.838ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
41682024-11-21T19:37:17.838ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
41692024-11-21T19:37:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
41702024-11-21T19:37:17.838ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
41712024-11-21T19:37:17.840ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 8.666946ms iteration = 9 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
41722024-11-21T19:37:17.841ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
41732024-11-21T19:37:17.841ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
41742024-11-21T19:37:17.841ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
41752024-11-21T19:37:17.841ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
41762024-11-21T19:37:17.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
41772024-11-21T19:37:17.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
41782024-11-21T19:37:17.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
41792024-11-21T19:37:17.842ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
41802024-11-21T19:37:17.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
41812024-11-21T19:37:17.843ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
41822024-11-21T19:37:17.844ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-11-21 19:27:17.831436462 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
41832024-11-21T19:37:17.844ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 13.117945ms iteration = 5 status = {"expiration":"2024-11-21T19:27:17.831436462Z","pruned":[]}
41842024-11-21T19:37:17.845ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
41852024-11-21T19:37:17.845ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
41862024-11-21T19:37:17.846ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
41872024-11-21T19:37:17.846ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
41882024-11-21T19:37:17.846ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
41892024-11-21T19:37:17.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
41902024-11-21T19:37:17.848ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
41912024-11-21T19:37:17.848ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
41922024-11-21T19:37:17.849ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
41932024-11-21T19:37:17.849ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
41942024-11-21T19:37:17.849ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
41952024-11-21T19:37:17.849ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
41962024-11-21T19:37:17.849ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
41972024-11-21T19:37:17.849ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
41982024-11-21T19:37:17.850ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
41992024-11-21T19:37:17.850ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
42002024-11-21T19:37:17.850ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
42012024-11-21T19:37:17.855ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 22.955636ms iteration = 5 status = {"errors":[],"found_port_ok":[]}
42022024-11-21T19:37:17.858ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
42032024-11-21T19:37:17.858ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
42042024-11-21T19:37:17.859ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
42052024-11-21T19:37:17.859ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
42062024-11-21T19:37:17.859ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
42072024-11-21T19:37:17.859ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
42082024-11-21T19:37:17.867ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
42092024-11-21T19:37:17.867ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
42102024-11-21T19:37:17.870ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 980, tv_nsec: 748122879 } })
42112024-11-21T19:37:17.871ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
42122024-11-21T19:37:17.902ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
42132024-11-21T19:37:17.902ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
42142024-11-21T19:37:17.905ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 73.132908ms iteration = 9 status = {"errors":[],"garbage_collect_requested":[]}
42152024-11-21T19:37:17.906ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
42162024-11-21T19:37:17.907ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
42172024-11-21T19:37:17.908ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
42182024-11-21T19:37:17.908ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0 total = 0
42192024-11-21T19:37:17.908ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:171 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0 vmms_found = 0
42202024-11-21T19:37:17.908ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 76.135637ms iteration = 5 status = {"errors":[],"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0,"vmms_found":0}
42212024-11-21T19:37:17.909ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
42222024-11-21T19:37:17.909ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
42232024-11-21T19:37:17.909ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
42242024-11-21T19:37:17.910ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
42252024-11-21T19:37:17.910ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
42262024-11-21T19:37:17.910ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
42272024-11-21T19:37:17.910ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
42282024-11-21T19:37:17.911ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
42292024-11-21T19:37:17.911ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 79.604604ms iteration = 5 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
42302024-11-21T19:37:17.911ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
42312024-11-21T19:37:17.911ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
42322024-11-21T19:37:17.911ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
42332024-11-21T19:37:17.911ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
42342024-11-21T19:37:17.911ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
42352024-11-21T19:37:17.913ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
42362024-11-21T19:37:17.914ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
42372024-11-21T19:37:17.914ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
42382024-11-21T19:37:17.915ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
42392024-11-21T19:37:17.916ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
42402024-11-21T19:37:17.916ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
42412024-11-21T19:37:17.916ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 84.458999ms iteration = 9 status = {"errors":[],"records_set_to_done":[]}
42422024-11-21T19:37:17.916ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
42432024-11-21T19:37:17.918ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
42442024-11-21T19:37:17.918ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
42452024-11-21T19:37:17.918ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
42462024-11-21T19:37:17.921ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
42472024-11-21T19:37:17.921ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 89.865561ms iteration = 9 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
42482024-11-21T19:37:17.922ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
42492024-11-21T19:37:17.924ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
42502024-11-21T19:37:17.924ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
42512024-11-21T19:37:17.924ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
42522024-11-21T19:37:17.925ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
42532024-11-21T19:37:17.925ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
42542024-11-21T19:37:17.925ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
42552024-11-21T19:37:17.928ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 96.111664ms iteration = 25 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
42562024-11-21T19:37:17.932ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
42572024-11-21T19:37:17.936ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
42582024-11-21T19:37:17.936ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
42592024-11-21T19:37:17.937ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
42602024-11-21T19:37:17.945ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
42612024-11-21T19:37:17.945ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
42622024-11-21T19:37:17.948ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
42632024-11-21T19:37:17.949ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
42642024-11-21T19:37:17.951ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
42652024-11-21T19:37:17.952ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
42662024-11-21T19:37:17.952ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 121.348771ms iteration = 25 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
42672024-11-21T19:37:17.954ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
42682024-11-21T19:37:17.954ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
42692024-11-21T19:37:17.954ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
42702024-11-21T19:37:17.954ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
42712024-11-21T19:37:17.954ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
42722024-11-21T19:37:17.954ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
42732024-11-21T19:37:17.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
42742024-11-21T19:37:17.954ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
42752024-11-21T19:37:17.957ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
42762024-11-21T19:37:17.957ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 126.318128ms iteration = 9 status = 0
42772024-11-21T19:37:17.957ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
42782024-11-21T19:37:17.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
42792024-11-21T19:37:17.958ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 126.989917ms iteration = 9 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
42802024-11-21T19:37:17.961ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
42812024-11-21T19:37:17.961ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
42822024-11-21T19:37:17.962ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
42832024-11-21T19:37:17.975ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
42842024-11-21T19:37:17.975ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
42852024-11-21T19:37:17.975ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
42862024-11-21T19:37:17.975ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
42872024-11-21T19:37:17.977ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 146.516867ms iteration = 10 status = {"physical_disks_added":0}
42882024-11-21T19:37:17.978ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
42892024-11-21T19:37:17.978ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
42902024-11-21T19:37:17.978ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
42912024-11-21T19:37:17.978ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
42922024-11-21T19:37:17.978ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
42932024-11-21T19:37:17.979ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
42942024-11-21T19:37:17.980ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
42952024-11-21T19:37:17.980ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
42962024-11-21T19:37:17.981ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
42972024-11-21T19:37:17.982ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 150.748334ms iteration = 9 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
42982024-11-21T19:37:17.984ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
42992024-11-21T19:37:17.984ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
43002024-11-21T19:37:17.985ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
43012024-11-21T19:37:17.985ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
43022024-11-21T19:37:17.985ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
43032024-11-21T19:37:17.985ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
43042024-11-21T19:37:17.988ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
43052024-11-21T19:37:17.988ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 156.424261ms iteration = 9 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
43062024-11-21T19:37:18.001ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
43072024-11-21T19:37:18.008ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
43082024-11-21T19:37:18.008ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
43092024-11-21T19:37:18.008ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
43102024-11-21T19:37:18.008ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
43112024-11-21T19:37:18.010ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
43122024-11-21T19:37:18.011ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
43132024-11-21T19:37:18.011ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
43142024-11-21T19:37:18.011ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 179.790322ms iteration = 9 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
43152024-11-21T19:37:18.013ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
43162024-11-21T19:37:18.013ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
43172024-11-21T19:37:18.013ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
43182024-11-21T19:37:18.016ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 51 version = 1
43192024-11-21T19:37:18.016ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
43202024-11-21T19:37:18.016ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-11-21 19:33:31.797390 UTC dns_group = internal generation = 1
43212024-11-21T19:37:18.016ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 185.182154ms iteration = 5 status = {"generation":1}
43222024-11-21T19:37:18.019ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
43232024-11-21T19:37:18.019ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
43242024-11-21T19:37:18.020ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "37724b2a-6a40-4737-b6e2-5ebb8503a62a", "content-length": "26", "date": "Thu, 21 Nov 2024 19:37:17 GMT"} })
43252024-11-21T19:37:18.020ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
43262024-11-21T19:37:18.021ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
43272024-11-21T19:37:18.022ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
43282024-11-21T19:37:18.022ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
43292024-11-21T19:37:18.023ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "4eb90284-a1de-4416-b0fd-ede7f44471c3", "content-length": "26", "date": "Thu, 21 Nov 2024 19:37:17 GMT"} })
43302024-11-21T19:37:18.023ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
43312024-11-21T19:37:18.025ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
43322024-11-21T19:37:18.027ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
43332024-11-21T19:37:18.027ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 0 version = 2
43342024-11-21T19:37:18.027ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
43352024-11-21T19:37:18.027ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
43362024-11-21T19:37:18.040ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-11-21T19:33:15.446964Z, time_modified: 2024-11-21T19:33:15.446964Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }) }
43372024-11-21T19:37:18.040ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 209.019863ms iteration = 5 status = {"by_dns_name":{"default-silo.sys.oxide.test":{"silo_id":"001de000-5110-4000-8000-000000000000","tls_certs":[]},"recovery.sys.oxide.test":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]}},"default_endpoint":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]},"warnings":["silo 001de000-5110-4000-8000-000000000000 with DNS name \\"default-silo.sys.oxide.test\\" has no usable certificates"]}
43382024-11-21T19:37:18.062ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
43392024-11-21T19:37:18.062ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
43402024-11-21T19:37:18.074ZWARNnexus (BackgroundTasks): instance reincarnation completed with errors background_task = instance_reincarnation file = nexus/src/app/background/tasks/instance_reincarnation.rs:103 instances_changed_state = 0 instances_found = 0 instances_reincarnated = 0 query_errors = 0 restart_errors = 0
43412024-11-21T19:37:18.074ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_reincarnation elapsed = 241.986764ms iteration = 5 status = {"changed_state":[],"disabled":false,"errors":[],"instances_found":{"Failed":0,"SagaUnwound":0},"instances_reincarnated":[],"restart_errors":[]}
43422024-11-21T19:37:18.081ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
43432024-11-21T19:37:18.176ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
43442024-11-21T19:37:18.176ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::8]:5353/config", status: 204, headers: {"x-request-id": "b6b779ad-f032-4485-8c4f-258d79b30260", "date": "Thu, 21 Nov 2024 19:37:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
43452024-11-21T19:37:18.176ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
43462024-11-21T19:37:18.177ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
43472024-11-21T19:37:18.177ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
43482024-11-21T19:37:18.177ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "9ab77d4d-d2e4-4b95-aa90-d06028067c93", "content-length": "26", "date": "Thu, 21 Nov 2024 19:37:17 GMT"} })
43492024-11-21T19:37:18.177ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
43502024-11-21T19:37:18.179ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
43512024-11-21T19:37:18.179ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
43522024-11-21T19:37:18.179ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::9]:5353/config", status: 204, headers: {"x-request-id": "87ef3aef-2405-4065-bbc6-97de3ad654aa", "date": "Thu, 21 Nov 2024 19:37:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
43532024-11-21T19:37:18.179ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
43542024-11-21T19:37:18.180ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
43552024-11-21T19:37:18.180ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
43562024-11-21T19:37:18.181ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
43572024-11-21T19:37:18.181ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
43582024-11-21T19:37:18.181ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 349.799624ms iteration = 7 status = {"generation":2,"server_results":{"[fd00:1122:3344:101::8]:5353":{"Ok":null},"[fd00:1122:3344:101::9]:5353":{"Ok":null}}}
43592024-11-21T19:37:18.183ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
43602024-11-21T19:37:18.183ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
43612024-11-21T19:37:18.183ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
43622024-11-21T19:37:18.184ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
43632024-11-21T19:37:18.184ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
43642024-11-21T19:37:18.184ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
43652024-11-21T19:37:18.184ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
43662024-11-21T19:37:18.186ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 1 version = 2
43672024-11-21T19:37:18.186ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
43682024-11-21T19:37:18.186ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-11-21 19:33:32.758954 UTC dns_group = external generation = 2
43692024-11-21T19:37:18.186ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 354.905442ms iteration = 5 status = {"generation":2}
43702024-11-21T19:37:18.186ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
43712024-11-21T19:37:18.186ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
43722024-11-21T19:37:18.187ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
43732024-11-21T19:37:18.191ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
43742024-11-21T19:37:18.191ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "47d4d671-3886-4486-a9fb-9a94f45a8902", "content-length": "1", "date": "Thu, 21 Nov 2024 19:37:17 GMT"} })
43752024-11-21T19:37:18.192ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 361.41158ms iteration = 9 status = {}
43762024-11-21T19:37:18.192ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
43772024-11-21T19:37:18.192ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
43782024-11-21T19:37:18.194ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 363.053891ms iteration = 9 status = null
43792024-11-21T19:37:18.219ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "ed61fa50-4f71-47d1-bb9a-1e5774b50a6d", "content-length": "372", "date": "Thu, 21 Nov 2024 19:37:18 GMT"} })
43802024-11-21T19:37:18.219ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
43812024-11-21T19:37:18.219ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
43822024-11-21T19:37:18.234ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
43832024-11-21T19:37:18.234ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
43842024-11-21T19:37:18.234ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "d5b5d5e2-c6ef-44de-bb7f-764c530baf17", "date": "Thu, 21 Nov 2024 19:37:18 GMT"} })
43852024-11-21T19:37:18.234ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
43862024-11-21T19:37:18.234ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
43872024-11-21T19:37:18.237ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
43882024-11-21T19:37:18.237ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
43892024-11-21T19:37:18.240ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
43902024-11-21T19:37:18.240ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
43912024-11-21T19:37:18.248ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
43922024-11-21T19:37:18.283ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
43932024-11-21T19:37:18.285ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
43942024-11-21T19:37:18.285ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
43952024-11-21T19:37:18.286ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "c64e103f-3ffa-44af-928c-6e97693655d0", "content-length": "691", "date": "Thu, 21 Nov 2024 19:37:18 GMT"} })
43962024-11-21T19:37:18.286ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
43972024-11-21T19:37:18.286ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
43982024-11-21T19:37:18.286ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
43992024-11-21T19:37:18.287ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "7721ba25-27a0-4380-b243-cad30dde97cc", "date": "Thu, 21 Nov 2024 19:37:18 GMT"} })
44002024-11-21T19:37:18.288ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 456.961016ms iteration = 9 status = {}
44012024-11-21T19:37:18.288ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 456.779164ms iteration = 9 status = {}
44022024-11-21T19:37:18.322ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
44032024-11-21T19:37:18.323ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:2::1]:5353/config", status: 204, headers: {"x-request-id": "ec48d53e-99ac-4b4c-81cf-4fc5c78ad9f2", "date": "Thu, 21 Nov 2024 19:37:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
44042024-11-21T19:37:18.323ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
44052024-11-21T19:37:18.324ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:1::1]:5353/config", status: 204, headers: {"x-request-id": "5f7df62e-ea80-4837-b89e-36d749baff8a", "date": "Thu, 21 Nov 2024 19:37:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
44062024-11-21T19:37:18.324ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
44072024-11-21T19:37:18.326ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:3::1]:5353/config", status: 204, headers: {"x-request-id": "80580911-da75-4bd5-b832-166f0a384520", "date": "Thu, 21 Nov 2024 19:37:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
44082024-11-21T19:37:18.326ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
44092024-11-21T19:37:18.328ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
44102024-11-21T19:37:18.328ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 497.100152ms iteration = 7 status = {"generation":1,"server_results":{"[fd00:1122:3344:1::1]:5353":{"Ok":null},"[fd00:1122:3344:2::1]:5353":{"Ok":null},"[fd00:1122:3344:3::1]:5353":{"Ok":null}}}
44112024-11-21T19:37:18.360ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
44122024-11-21T19:37:18.360ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
44132024-11-21T19:37:18.362ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
44142024-11-21T19:37:18.362ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
44152024-11-21T19:37:18.362ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
44162024-11-21T19:37:18.372ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
44172024-11-21T19:37:18.372ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "46bab610-d50b-45a6-96ea-b6ae958930e8", "content-length": "134", "date": "Thu, 21 Nov 2024 19:37:18 GMT"} })
44182024-11-21T19:37:18.372ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
44192024-11-21T19:37:18.372ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
44202024-11-21T19:37:18.372ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
44212024-11-21T19:37:18.372ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
44222024-11-21T19:37:18.372ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
44232024-11-21T19:37:18.373ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
44242024-11-21T19:37:18.373ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
44252024-11-21T19:37:18.373ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
44262024-11-21T19:37:18.373ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
44272024-11-21T19:37:18.373ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
44282024-11-21T19:37:18.373ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
44292024-11-21T19:37:18.373ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
44302024-11-21T19:37:18.373ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
44312024-11-21T19:37:18.375ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
44322024-11-21T19:37:18.375ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "d4899a2e-340f-4175-b666-7d3e4bb7608d", "content-length": "2", "date": "Thu, 21 Nov 2024 19:37:18 GMT"} })
44332024-11-21T19:37:18.375ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
44342024-11-21T19:37:18.375ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "89523969-80a3-4753-b38b-a654460834dc", "content-length": "51", "date": "Thu, 21 Nov 2024 19:37:18 GMT"} })
44352024-11-21T19:37:18.375ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
44362024-11-21T19:37:18.375ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
44372024-11-21T19:37:18.462ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "9f2fb5f7-7a90-484b-a3ec-3cece231fa49", "date": "Thu, 21 Nov 2024 19:37:17 GMT"} })
44382024-11-21T19:37:18.462ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
44392024-11-21T19:37:18.462ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
44402024-11-21T19:37:18.467ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "9d8b3458-ce4a-42ae-badd-e3c2b376f0b5", "content-length": "682", "date": "Thu, 21 Nov 2024 19:37:17 GMT"} })
44412024-11-21T19:37:18.467ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
44422024-11-21T19:37:18.467ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
44432024-11-21T19:37:18.469ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
44442024-11-21T19:37:18.470ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
44452024-11-21T19:37:18.473ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
44462024-11-21T19:37:18.473ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
44472024-11-21T19:37:18.475ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
44482024-11-21T19:37:18.475ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
44492024-11-21T19:37:18.483ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 652.122144ms iteration = 9 status = {}
44502024-11-21T19:37:26.960ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 513 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 2124a8bb-00f4-404c-b17d-c6fd89a2c68a response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
44512024-11-21T19:37:27.831ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 26 reason = Timeout
44522024-11-21T19:37:27.831ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
44532024-11-21T19:37:27.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
44542024-11-21T19:37:27.832ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 26 reason = Timeout
44552024-11-21T19:37:27.832ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
44562024-11-21T19:37:27.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
44572024-11-21T19:37:27.833ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
44582024-11-21T19:37:27.834ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
44592024-11-21T19:37:27.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
44602024-11-21T19:37:27.837ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
44612024-11-21T19:37:27.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
44622024-11-21T19:37:27.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
44632024-11-21T19:37:27.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
44642024-11-21T19:37:27.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
44652024-11-21T19:37:27.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
44662024-11-21T19:37:27.845ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
44672024-11-21T19:37:27.845ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
44682024-11-21T19:37:27.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
44692024-11-21T19:37:27.862ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
44702024-11-21T19:37:27.862ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 31.105815ms iteration = 26 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
44712024-11-21T19:37:27.862ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 30.636159ms iteration = 26 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
44722024-11-21T19:37:37.831ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 27 reason = Timeout
44732024-11-21T19:37:37.831ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
44742024-11-21T19:37:37.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
44752024-11-21T19:37:37.832ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 27 reason = Timeout
44762024-11-21T19:37:37.832ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
44772024-11-21T19:37:37.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
44782024-11-21T19:37:37.833ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
44792024-11-21T19:37:37.834ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
44802024-11-21T19:37:37.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
44812024-11-21T19:37:37.836ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 3.671198ms iteration = 27 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
44822024-11-21T19:37:37.836ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
44832024-11-21T19:37:37.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
44842024-11-21T19:37:37.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
44852024-11-21T19:37:37.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
44862024-11-21T19:37:37.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
44872024-11-21T19:37:37.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
44882024-11-21T19:37:37.843ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
44892024-11-21T19:37:37.843ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
44902024-11-21T19:37:37.843ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
44912024-11-21T19:37:37.852ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
44922024-11-21T19:37:37.852ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 20.630696ms iteration = 27 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
44932024-11-21T19:37:41.959ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 328 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 00b048f7-e74a-4204-9842-521c8220f7b7 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
44942024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 10 reason = Timeout
44952024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
44962024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 28 reason = Timeout
44972024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 10 reason = Timeout
44982024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
44992024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
45002024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 11 reason = Timeout
45012024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
45022024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 10 reason = Timeout
45032024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 10 reason = Timeout
45042024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
45052024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 10 reason = Timeout
45062024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
45072024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 10 reason = Timeout
45082024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 10 reason = Timeout
45092024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
45102024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
45112024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
45122024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
45132024-11-21T19:37:47.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
45142024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
45152024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
45162024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
45172024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
45182024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
45192024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 10 reason = Timeout
45202024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 10 reason = Timeout
45212024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
45222024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
45232024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 10 reason = Timeout
45242024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
45252024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
45262024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 10 reason = Timeout
45272024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
45282024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 28 reason = Timeout
45292024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
45302024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 10 reason = Timeout
45312024-11-21T19:37:47.832ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
45322024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 10 reason = Timeout
45332024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
45342024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 10 reason = Timeout
45352024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
45362024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
45372024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
45382024-11-21T19:37:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
45392024-11-21T19:37:47.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
45402024-11-21T19:37:47.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
45412024-11-21T19:37:47.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
45422024-11-21T19:37:47.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
45432024-11-21T19:37:47.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
45442024-11-21T19:37:47.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
45452024-11-21T19:37:47.833ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
45462024-11-21T19:37:47.835ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1010, tv_nsec: 712284850 } })
45472024-11-21T19:37:47.835ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.894352ms iteration = 10 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
45482024-11-21T19:37:47.835ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
45492024-11-21T19:37:47.835ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
45502024-11-21T19:37:47.835ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1010, tv_nsec: 712842548 } })
45512024-11-21T19:37:47.835ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
45522024-11-21T19:37:47.836ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
45532024-11-21T19:37:47.837ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
45542024-11-21T19:37:47.837ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
45552024-11-21T19:37:47.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
45562024-11-21T19:37:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
45572024-11-21T19:37:47.838ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
45582024-11-21T19:37:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
45592024-11-21T19:37:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
45602024-11-21T19:37:47.839ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
45612024-11-21T19:37:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
45622024-11-21T19:37:47.840ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
45632024-11-21T19:37:47.841ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
45642024-11-21T19:37:47.841ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
45652024-11-21T19:37:47.841ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
45662024-11-21T19:37:47.841ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
45672024-11-21T19:37:47.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
45682024-11-21T19:37:47.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
45692024-11-21T19:37:47.842ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
45702024-11-21T19:37:47.842ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 9.553358ms iteration = 10 status = {"errors":[],"garbage_collect_requested":[]}
45712024-11-21T19:37:47.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
45722024-11-21T19:37:47.842ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
45732024-11-21T19:37:47.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
45742024-11-21T19:37:47.844ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
45752024-11-21T19:37:47.844ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
45762024-11-21T19:37:47.844ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
45772024-11-21T19:37:47.844ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
45782024-11-21T19:37:47.844ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
45792024-11-21T19:37:47.844ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
45802024-11-21T19:37:47.847ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
45812024-11-21T19:37:47.847ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
45822024-11-21T19:37:47.847ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
45832024-11-21T19:37:47.847ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
45842024-11-21T19:37:47.847ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
45852024-11-21T19:37:47.848ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
45862024-11-21T19:37:47.848ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
45872024-11-21T19:37:47.848ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
45882024-11-21T19:37:47.848ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
45892024-11-21T19:37:47.853ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
45902024-11-21T19:37:47.853ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
45912024-11-21T19:37:47.853ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
45922024-11-21T19:37:47.853ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1010, tv_nsec: 730678736 } })
45932024-11-21T19:37:47.854ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
45942024-11-21T19:37:47.854ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
45952024-11-21T19:37:47.854ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
45962024-11-21T19:37:47.855ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
45972024-11-21T19:37:47.859ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
45982024-11-21T19:37:47.859ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
45992024-11-21T19:37:47.859ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
46002024-11-21T19:37:47.859ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
46012024-11-21T19:37:47.859ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
46022024-11-21T19:37:47.859ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
46032024-11-21T19:37:47.860ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
46042024-11-21T19:37:47.861ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
46052024-11-21T19:37:47.861ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 29.263961ms iteration = 10 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
46062024-11-21T19:37:47.864ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
46072024-11-21T19:37:47.864ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
46082024-11-21T19:37:47.864ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
46092024-11-21T19:37:47.882ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
46102024-11-21T19:37:47.883ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
46112024-11-21T19:37:47.883ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 51.070813ms iteration = 10 status = {"errors":[],"records_set_to_done":[]}
46122024-11-21T19:37:47.884ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
46132024-11-21T19:37:47.884ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 52.739585ms iteration = 28 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
46142024-11-21T19:37:47.884ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
46152024-11-21T19:37:47.885ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
46162024-11-21T19:37:47.887ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 54.429327ms iteration = 28 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
46172024-11-21T19:37:47.894ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
46182024-11-21T19:37:47.895ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
46192024-11-21T19:37:47.895ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 63.494459ms iteration = 10 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
46202024-11-21T19:37:47.904ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
46212024-11-21T19:37:47.904ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
46222024-11-21T19:37:47.904ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 72.275455ms iteration = 10 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
46232024-11-21T19:37:47.913ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 81.779943ms iteration = 11 status = {"physical_disks_added":0}
46242024-11-21T19:37:47.915ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
46252024-11-21T19:37:47.915ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
46262024-11-21T19:37:47.915ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "d6a6f5f1-8723-4f0c-b23a-4a3226aba0d0", "content-length": "26", "date": "Thu, 21 Nov 2024 19:37:47 GMT"} })
46272024-11-21T19:37:47.915ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
46282024-11-21T19:37:47.917ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
46292024-11-21T19:37:47.948ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
46302024-11-21T19:37:47.948ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
46312024-11-21T19:37:47.948ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
46322024-11-21T19:37:47.948ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
46332024-11-21T19:37:47.949ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "8d36ae55-8281-4b9f-9dd9-1d563538f335", "content-length": "26", "date": "Thu, 21 Nov 2024 19:37:47 GMT"} })
46342024-11-21T19:37:47.949ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
46352024-11-21T19:37:47.950ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
46362024-11-21T19:37:47.963ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
46372024-11-21T19:37:47.963ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
46382024-11-21T19:37:47.964ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "0d807344-2268-4e70-94dd-a27906f8ee2d", "content-length": "26", "date": "Thu, 21 Nov 2024 19:37:47 GMT"} })
46392024-11-21T19:37:47.964ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
46402024-11-21T19:37:47.965ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
46412024-11-21T19:37:48.016ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
46422024-11-21T19:37:48.016ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
46432024-11-21T19:37:48.019ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
46442024-11-21T19:37:48.019ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
46452024-11-21T19:37:48.021ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 190.050399ms iteration = 10 status = {}
46462024-11-21T19:37:48.035ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
46472024-11-21T19:37:48.035ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "27488f53-974b-4ac6-8ab2-79ef95b32887", "content-length": "1", "date": "Thu, 21 Nov 2024 19:37:47 GMT"} })
46482024-11-21T19:37:48.036ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
46492024-11-21T19:37:48.036ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
46502024-11-21T19:37:48.038ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 206.854964ms iteration = 10 status = null
46512024-11-21T19:37:48.046ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 213.812717ms iteration = 10 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
46522024-11-21T19:37:48.048ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
46532024-11-21T19:37:48.048ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
46542024-11-21T19:37:48.049ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
46552024-11-21T19:37:48.049ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 216.924048ms iteration = 10 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
46562024-11-21T19:37:48.052ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
46572024-11-21T19:37:48.052ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
46582024-11-21T19:37:48.052ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
46592024-11-21T19:37:48.054ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 222.626154ms iteration = 10 status = 0
46602024-11-21T19:37:48.059ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "f769ce65-eaca-41a2-9d05-3ccf79823d59", "content-length": "372", "date": "Thu, 21 Nov 2024 19:37:48 GMT"} })
46612024-11-21T19:37:48.059ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
46622024-11-21T19:37:48.059ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
46632024-11-21T19:37:48.073ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
46642024-11-21T19:37:48.073ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
46652024-11-21T19:37:48.094ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
46662024-11-21T19:37:48.094ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
46672024-11-21T19:37:48.096ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
46682024-11-21T19:37:48.096ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
46692024-11-21T19:37:48.096ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
46702024-11-21T19:37:48.106ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
46712024-11-21T19:37:48.107ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "0bef06a2-33e5-4bbf-b97d-fb6a774cbfe1", "content-length": "134", "date": "Thu, 21 Nov 2024 19:37:47 GMT"} })
46722024-11-21T19:37:48.107ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
46732024-11-21T19:37:48.107ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
46742024-11-21T19:37:48.107ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
46752024-11-21T19:37:48.107ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
46762024-11-21T19:37:48.107ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
46772024-11-21T19:37:48.107ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
46782024-11-21T19:37:48.107ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
46792024-11-21T19:37:48.107ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
46802024-11-21T19:37:48.107ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
46812024-11-21T19:37:48.107ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
46822024-11-21T19:37:48.108ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
46832024-11-21T19:37:48.108ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
46842024-11-21T19:37:48.108ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
46852024-11-21T19:37:48.110ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
46862024-11-21T19:37:48.111ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
46872024-11-21T19:37:48.111ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
46882024-11-21T19:37:48.111ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "8887aa2e-f3b9-43a5-9598-cfa6dfab376c", "content-length": "2", "date": "Thu, 21 Nov 2024 19:37:47 GMT"} })
46892024-11-21T19:37:48.111ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
46902024-11-21T19:37:48.111ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "37e83a09-3258-4baa-9309-5b2d608b1d3b", "content-length": "51", "date": "Thu, 21 Nov 2024 19:37:47 GMT"} })
46912024-11-21T19:37:48.111ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
46922024-11-21T19:37:48.111ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
46932024-11-21T19:37:48.113ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
46942024-11-21T19:37:48.115ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
46952024-11-21T19:37:48.115ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
46962024-11-21T19:37:48.115ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
46972024-11-21T19:37:48.147ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
46982024-11-21T19:37:48.149ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
46992024-11-21T19:37:48.149ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
47002024-11-21T19:37:48.150ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
47012024-11-21T19:37:48.167ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "4c938dca-603f-43f8-b8be-90b5c7b71118", "date": "Thu, 21 Nov 2024 19:37:47 GMT"} })
47022024-11-21T19:37:48.167ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
47032024-11-21T19:37:48.167ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
47042024-11-21T19:37:48.170ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
47052024-11-21T19:37:48.170ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
47062024-11-21T19:37:48.174ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
47072024-11-21T19:37:48.175ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
47082024-11-21T19:37:48.177ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
47092024-11-21T19:37:48.177ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
47102024-11-21T19:37:48.178ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "5626e2bd-7d00-43c6-a0b2-48bf9ae50c16", "date": "Thu, 21 Nov 2024 19:37:47 GMT"} })
47112024-11-21T19:37:48.180ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 347.425362ms iteration = 10 status = {}
47122024-11-21T19:37:48.184ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
47132024-11-21T19:37:48.186ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "8056d10d-dac2-47f4-bec1-ebf46d5fae48", "content-length": "691", "date": "Thu, 21 Nov 2024 19:37:47 GMT"} })
47142024-11-21T19:37:48.186ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
47152024-11-21T19:37:48.186ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
47162024-11-21T19:37:48.186ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
47172024-11-21T19:37:48.187ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 356.259541ms iteration = 10 status = {}
47182024-11-21T19:37:48.257ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "9372dd03-bf3a-408c-a371-732ab4d80f65", "date": "Thu, 21 Nov 2024 19:37:48 GMT"} })
47192024-11-21T19:37:48.257ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
47202024-11-21T19:37:48.257ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
47212024-11-21T19:37:48.257ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "02d16502-81f3-478d-87ca-6ce900060582", "content-length": "682", "date": "Thu, 21 Nov 2024 19:37:48 GMT"} })
47222024-11-21T19:37:48.257ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
47232024-11-21T19:37:48.258ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
47242024-11-21T19:37:48.261ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
47252024-11-21T19:37:48.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
47262024-11-21T19:37:48.265ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
47272024-11-21T19:37:48.265ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
47282024-11-21T19:37:48.267ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
47292024-11-21T19:37:48.267ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
47302024-11-21T19:37:48.273ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 442.359431ms iteration = 10 status = {}
47312024-11-21T19:37:56.961ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 313 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 11479943-de70-4c60-8847-e42ada47782f response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
47322024-11-21T19:37:57.831ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 29 reason = Timeout
47332024-11-21T19:37:57.831ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
47342024-11-21T19:37:57.831ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
47352024-11-21T19:37:57.832ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 29 reason = Timeout
47362024-11-21T19:37:57.832ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
47372024-11-21T19:37:57.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
47382024-11-21T19:37:57.833ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
47392024-11-21T19:37:57.836ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
47402024-11-21T19:37:57.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
47412024-11-21T19:37:57.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
47422024-11-21T19:37:57.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
47432024-11-21T19:37:57.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
47442024-11-21T19:37:57.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
47452024-11-21T19:37:57.843ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
47462024-11-21T19:37:57.843ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
47472024-11-21T19:37:57.843ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
47482024-11-21T19:37:57.854ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
47492024-11-21T19:37:57.854ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 23.304162ms iteration = 29 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
47502024-11-21T19:37:57.860ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
47512024-11-21T19:37:57.861ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
47522024-11-21T19:37:57.862ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 30.280125ms iteration = 29 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
47532024-11-21T19:38:01.952ZINFOnexus (dropshot_internal): accepted connection file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023 local_addr = [fd00:1122:3344:101::c]:12221 remote_addr = [fd00:1122:3344:101::d]:56016
47542024-11-21T19:38:01.952ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = 03efb69a-41e0-4a08-9f5b-148260b332ee roles = RoleSet { roles: {} } uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
47552024-11-21T19:38:01.952ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = 03efb69a-41e0-4a08-9f5b-148260b332ee resource = Database result = Ok(()) uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
47562024-11-21T19:38:01.955ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 3329 local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = 03efb69a-41e0-4a08-9f5b-148260b332ee response_code = 200 uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
47572024-11-21T19:38:01.956ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = 3931f041-9676-4351-8e3f-0698f33bc6bc roles = RoleSet { roles: {} } uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
47582024-11-21T19:38:01.956ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = 3931f041-9676-4351-8e3f-0698f33bc6bc resource = Database result = Ok(()) uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
47592024-11-21T19:38:01.958ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 2895 local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = 3931f041-9676-4351-8e3f-0698f33bc6bc response_code = 200 uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
47602024-11-21T19:38:07.832ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 30 reason = Timeout
47612024-11-21T19:38:07.832ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
47622024-11-21T19:38:07.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
47632024-11-21T19:38:07.832ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 30 reason = Timeout
47642024-11-21T19:38:07.832ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
47652024-11-21T19:38:07.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
47662024-11-21T19:38:07.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
47672024-11-21T19:38:07.836ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
47682024-11-21T19:38:07.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
47692024-11-21T19:38:07.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
47702024-11-21T19:38:07.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
47712024-11-21T19:38:07.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
47722024-11-21T19:38:07.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
47732024-11-21T19:38:07.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
47742024-11-21T19:38:07.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
47752024-11-21T19:38:07.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
47762024-11-21T19:38:07.851ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
47772024-11-21T19:38:07.851ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 19.594862ms iteration = 30 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
47782024-11-21T19:38:07.861ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
47792024-11-21T19:38:07.861ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
47802024-11-21T19:38:07.888ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 56.378104ms iteration = 30 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
47812024-11-21T19:38:11.961ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 329 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 907f939d-3b37-4259-8421-1ff312fe646c response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
47822024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 11 reason = Timeout
47832024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
47842024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 31 reason = Timeout
47852024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 6 reason = Timeout
47862024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
47872024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
47882024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
47892024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 8 reason = Timeout
47902024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 7 reason = Timeout
47912024-11-21T19:38:17.832ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:83 reason = blueprint disabled target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb
47922024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 11 reason = Timeout
47932024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 16.35\u{b5}s iteration = 7 status = {"enabled":false,"target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb"}
47942024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
47952024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 11 reason = Timeout
47962024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
47972024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 6 reason = Timeout
47982024-11-21T19:38:17.832ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-11-21 19:28:17.832344400 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
47992024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 6 reason = Timeout
48002024-11-21T19:38:17.832ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
48012024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
48022024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
48032024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 6 reason = Timeout
48042024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 11 reason = Timeout
48052024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 12 reason = Timeout
48062024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 6 reason = Timeout
48072024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
48082024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
48092024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 11 reason = Timeout
48102024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
48112024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 11 reason = Timeout
48122024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
48132024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
48142024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 6 reason = Timeout
48152024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 11 reason = Timeout
48162024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
48172024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 8 reason = Timeout
48182024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 6 reason = Timeout
48192024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
48202024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
48212024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
48222024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
48232024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
48242024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
48252024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
48262024-11-21T19:38:17.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
48272024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
48282024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
48292024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
48302024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 31 reason = Timeout
48312024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 11 reason = Timeout
48322024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 11 reason = Timeout
48332024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 11 reason = Timeout
48342024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
48352024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
48362024-11-21T19:38:17.833ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
48372024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
48382024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
48392024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
48402024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
48412024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 11 reason = Timeout
48422024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 11 reason = Timeout
48432024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
48442024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
48452024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
48462024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
48472024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 6 reason = Timeout
48482024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): activating background_task = service_firewall_rule_propagation iteration = 2 reason = Timeout
48492024-11-21T19:38:17.833ZDEBGnexus (service-firewall-rule-progator): starting background task for service firewall rule propagation background_task = service_firewall_rule_propagation
48502024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): activating background_task = instance_reincarnation iteration = 6 reason = Timeout
48512024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
48522024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
48532024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 11 reason = Timeout
48542024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
48552024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
48562024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 6 reason = Timeout
48572024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
48582024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 11 reason = Timeout
48592024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
48602024-11-21T19:38:17.833ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("4e669ff4-6dde-4cd2-8491-623460b56d8d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 1040, tv_nsec: 709707356 } })
48612024-11-21T19:38:17.833ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0377cf96-baf3-4e09-a570-eb7c67037d4d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0d1b3b31-4591-4bcb-931d-e8e03365f0d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:1::1))) }], valid_until: Instant { tv_sec: 1040, tv_nsec: 709722616 } })
48622024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
48632024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
48642024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
48652024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
48662024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
48672024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
48682024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
48692024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
48702024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
48712024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
48722024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
48732024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
48742024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
48752024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } dns_group = internal
48762024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
48772024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 1.49043ms iteration = 6 status = {"addresses":["[fd00:1122:3344:3::1]:5353","[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353"]}
48782024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 1.403729ms iteration = 6 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
48792024-11-21T19:38:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
48802024-11-21T19:38:17.838ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1040, tv_nsec: 714965476 } })
48812024-11-21T19:38:17.838ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 6.328485ms iteration = 11 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
48822024-11-21T19:38:17.838ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
48832024-11-21T19:38:17.838ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
48842024-11-21T19:38:17.839ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
48852024-11-21T19:38:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
48862024-11-21T19:38:17.839ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1040, tv_nsec: 716349485 } })
48872024-11-21T19:38:17.840ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
48882024-11-21T19:38:17.841ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
48892024-11-21T19:38:17.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
48902024-11-21T19:38:17.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
48912024-11-21T19:38:17.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
48922024-11-21T19:38:17.844ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
48932024-11-21T19:38:17.844ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
48942024-11-21T19:38:17.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
48952024-11-21T19:38:17.845ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-11-21 19:28:17.832344400 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
48962024-11-21T19:38:17.845ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 13.143246ms iteration = 6 status = {"expiration":"2024-11-21T19:28:17.832344400Z","pruned":[]}
48972024-11-21T19:38:17.848ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
48982024-11-21T19:38:17.851ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
48992024-11-21T19:38:17.851ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
49002024-11-21T19:38:17.851ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
49012024-11-21T19:38:17.852ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
49022024-11-21T19:38:17.852ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
49032024-11-21T19:38:17.852ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
49042024-11-21T19:38:17.852ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
49052024-11-21T19:38:17.854ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 21.409586ms iteration = 6 status = {"errors":[],"found_port_ok":[]}
49062024-11-21T19:38:17.857ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
49072024-11-21T19:38:17.857ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
49082024-11-21T19:38:17.858ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
49092024-11-21T19:38:17.861ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
49102024-11-21T19:38:17.861ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
49112024-11-21T19:38:17.861ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
49122024-11-21T19:38:17.864ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
49132024-11-21T19:38:17.864ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
49142024-11-21T19:38:17.869ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
49152024-11-21T19:38:17.869ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
49162024-11-21T19:38:17.870ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
49172024-11-21T19:38:17.870ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
49182024-11-21T19:38:17.878ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
49192024-11-21T19:38:17.878ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
49202024-11-21T19:38:17.878ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
49212024-11-21T19:38:17.879ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
49222024-11-21T19:38:17.880ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
49232024-11-21T19:38:17.880ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
49242024-11-21T19:38:17.883ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
49252024-11-21T19:38:17.887ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
49262024-11-21T19:38:17.887ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
49272024-11-21T19:38:17.888ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
49282024-11-21T19:38:17.890ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 57.908423ms iteration = 11 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
49292024-11-21T19:38:17.892ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
49302024-11-21T19:38:17.892ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 59.84296ms iteration = 11 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
49312024-11-21T19:38:17.893ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
49322024-11-21T19:38:17.895ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
49332024-11-21T19:38:17.895ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
49342024-11-21T19:38:17.895ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 62.656548ms iteration = 11 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
49352024-11-21T19:38:17.896ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
49362024-11-21T19:38:17.896ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
49372024-11-21T19:38:17.896ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
49382024-11-21T19:38:17.897ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
49392024-11-21T19:38:17.897ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
49402024-11-21T19:38:17.898ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
49412024-11-21T19:38:17.899ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
49422024-11-21T19:38:17.900ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
49432024-11-21T19:38:17.900ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
49442024-11-21T19:38:17.901ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
49452024-11-21T19:38:17.902ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
49462024-11-21T19:38:17.902ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
49472024-11-21T19:38:17.902ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
49482024-11-21T19:38:17.905ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
49492024-11-21T19:38:17.905ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
49502024-11-21T19:38:17.906ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1040, tv_nsec: 783307479 } })
49512024-11-21T19:38:17.907ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
49522024-11-21T19:38:17.908ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
49532024-11-21T19:38:17.909ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
49542024-11-21T19:38:17.910ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
49552024-11-21T19:38:17.914ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
49562024-11-21T19:38:17.914ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
49572024-11-21T19:38:17.914ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
49582024-11-21T19:38:17.914ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
49592024-11-21T19:38:17.915ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
49602024-11-21T19:38:17.916ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
49612024-11-21T19:38:17.916ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
49622024-11-21T19:38:17.916ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
49632024-11-21T19:38:17.916ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
49642024-11-21T19:38:17.918ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
49652024-11-21T19:38:17.918ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
49662024-11-21T19:38:17.918ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
49672024-11-21T19:38:17.920ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
49682024-11-21T19:38:17.922ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
49692024-11-21T19:38:17.922ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
49702024-11-21T19:38:17.922ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
49712024-11-21T19:38:17.922ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
49722024-11-21T19:38:17.923ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
49732024-11-21T19:38:17.923ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
49742024-11-21T19:38:17.923ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
49752024-11-21T19:38:17.924ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
49762024-11-21T19:38:17.925ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
49772024-11-21T19:38:17.925ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
49782024-11-21T19:38:17.926ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
49792024-11-21T19:38:17.927ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
49802024-11-21T19:38:17.927ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
49812024-11-21T19:38:17.928ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
49822024-11-21T19:38:17.928ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
49832024-11-21T19:38:17.928ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 96.133135ms iteration = 31 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
49842024-11-21T19:38:17.930ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0 total = 0
49852024-11-21T19:38:17.930ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:171 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0 vmms_found = 0
49862024-11-21T19:38:17.930ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 97.022157ms iteration = 6 status = {"errors":[],"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0,"vmms_found":0}
49872024-11-21T19:38:17.931ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 98.361374ms iteration = 11 status = {"errors":[],"garbage_collect_requested":[]}
49882024-11-21T19:38:17.932ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
49892024-11-21T19:38:17.932ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
49902024-11-21T19:38:17.932ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
49912024-11-21T19:38:17.932ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
49922024-11-21T19:38:17.934ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
49932024-11-21T19:38:17.935ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 102.423429ms iteration = 11 status = {"errors":[],"records_set_to_done":[]}
49942024-11-21T19:38:17.935ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
49952024-11-21T19:38:17.935ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
49962024-11-21T19:38:17.936ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
49972024-11-21T19:38:17.936ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
49982024-11-21T19:38:17.936ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
49992024-11-21T19:38:17.936ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
50002024-11-21T19:38:17.936ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
50012024-11-21T19:38:17.936ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
50022024-11-21T19:38:17.936ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 104.561867ms iteration = 6 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
50032024-11-21T19:38:17.938ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
50042024-11-21T19:38:17.938ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
50052024-11-21T19:38:17.938ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
50062024-11-21T19:38:17.939ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
50072024-11-21T19:38:17.946ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
50082024-11-21T19:38:17.946ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
50092024-11-21T19:38:17.946ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
50102024-11-21T19:38:17.950ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
50112024-11-21T19:38:17.950ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
50122024-11-21T19:38:17.950ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
50132024-11-21T19:38:17.952ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
50142024-11-21T19:38:17.952ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
50152024-11-21T19:38:17.955ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
50162024-11-21T19:38:17.955ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
50172024-11-21T19:38:17.957ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
50182024-11-21T19:38:17.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
50192024-11-21T19:38:17.962ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
50202024-11-21T19:38:17.962ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
50212024-11-21T19:38:17.964ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
50222024-11-21T19:38:17.965ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
50232024-11-21T19:38:17.965ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
50242024-11-21T19:38:17.965ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
50252024-11-21T19:38:17.967ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
50262024-11-21T19:38:17.967ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
50272024-11-21T19:38:17.968ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
50282024-11-21T19:38:17.969ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
50292024-11-21T19:38:17.969ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
50302024-11-21T19:38:17.971ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
50312024-11-21T19:38:17.972ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
50322024-11-21T19:38:17.973ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
50332024-11-21T19:38:17.975ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
50342024-11-21T19:38:17.975ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
50352024-11-21T19:38:17.975ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
50362024-11-21T19:38:17.976ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
50372024-11-21T19:38:17.976ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
50382024-11-21T19:38:17.976ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
50392024-11-21T19:38:17.979ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 146.432847ms iteration = 31 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
50402024-11-21T19:38:17.982ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
50412024-11-21T19:38:17.986ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
50422024-11-21T19:38:17.986ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
50432024-11-21T19:38:17.987ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
50442024-11-21T19:38:17.989ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 156.670823ms iteration = 11 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
50452024-11-21T19:38:17.996ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
50462024-11-21T19:38:17.998ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
50472024-11-21T19:38:17.998ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 0 version = 2
50482024-11-21T19:38:17.998ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
50492024-11-21T19:38:17.998ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
50502024-11-21T19:38:18.006ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
50512024-11-21T19:38:18.006ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
50522024-11-21T19:38:18.007ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "98063ab1-73f4-4d06-9337-5c9f18218ae3", "content-length": "26", "date": "Thu, 21 Nov 2024 19:38:17 GMT"} })
50532024-11-21T19:38:18.007ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
50542024-11-21T19:38:18.010ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
50552024-11-21T19:38:18.010ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 177.304539ms iteration = 11 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
50562024-11-21T19:38:18.011ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
50572024-11-21T19:38:18.011ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
50582024-11-21T19:38:18.011ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
50592024-11-21T19:38:18.011ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
50602024-11-21T19:38:18.011ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
50612024-11-21T19:38:18.022ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
50622024-11-21T19:38:18.022ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
50632024-11-21T19:38:18.022ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "209ed623-c65e-49cb-b3d2-12324dad0ffb", "content-length": "26", "date": "Thu, 21 Nov 2024 19:38:17 GMT"} })
50642024-11-21T19:38:18.022ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
50652024-11-21T19:38:18.022ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
50662024-11-21T19:38:18.025ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
50672024-11-21T19:38:18.053ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
50682024-11-21T19:38:18.053ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
50692024-11-21T19:38:18.073ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 241.221803ms iteration = 12 status = {"physical_disks_added":0}
50702024-11-21T19:38:18.105ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
50712024-11-21T19:38:18.105ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
50722024-11-21T19:38:18.105ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "3bf65490-5a27-4f18-86be-f9cb1961fa5e", "content-length": "26", "date": "Thu, 21 Nov 2024 19:38:17 GMT"} })
50732024-11-21T19:38:18.105ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
50742024-11-21T19:38:18.107ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
50752024-11-21T19:38:18.124ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
50762024-11-21T19:38:18.124ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
50772024-11-21T19:38:18.124ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "92741ae8-7cfe-4186-a894-bfcc5680788b", "content-length": "372", "date": "Thu, 21 Nov 2024 19:38:17 GMT"} })
50782024-11-21T19:38:18.124ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
50792024-11-21T19:38:18.125ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
50802024-11-21T19:38:18.127ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
50812024-11-21T19:38:18.127ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
50822024-11-21T19:38:18.127ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 295.341147ms iteration = 11 status = 0
50832024-11-21T19:38:18.127ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
50842024-11-21T19:38:18.128ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
50852024-11-21T19:38:18.131ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
50862024-11-21T19:38:18.131ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
50872024-11-21T19:38:18.131ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
50882024-11-21T19:38:18.131ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
50892024-11-21T19:38:18.131ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
50902024-11-21T19:38:18.131ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
50912024-11-21T19:38:18.138ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
50922024-11-21T19:38:18.138ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 1 version = 2
50932024-11-21T19:38:18.138ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
50942024-11-21T19:38:18.138ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-11-21 19:33:32.758954 UTC dns_group = external generation = 2
50952024-11-21T19:38:18.138ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 306.452094ms iteration = 6 status = {"generation":2}
50962024-11-21T19:38:18.138ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
50972024-11-21T19:38:18.141ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
50982024-11-21T19:38:18.141ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
50992024-11-21T19:38:18.144ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
51002024-11-21T19:38:18.145ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
51012024-11-21T19:38:18.171ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
51022024-11-21T19:38:18.171ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
51032024-11-21T19:38:18.174ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
51042024-11-21T19:38:18.175ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
51052024-11-21T19:38:18.175ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
51062024-11-21T19:38:18.175ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
51072024-11-21T19:38:18.175ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
51082024-11-21T19:38:18.176ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
51092024-11-21T19:38:18.176ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::9]:5353/config", status: 204, headers: {"x-request-id": "6e27647c-a6c2-41fb-adde-cf45db446f0f", "date": "Thu, 21 Nov 2024 19:38:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
51102024-11-21T19:38:18.176ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
51112024-11-21T19:38:18.178ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
51122024-11-21T19:38:18.178ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::8]:5353/config", status: 204, headers: {"x-request-id": "56f0e310-cd9a-4762-9bbb-4925e050bb44", "date": "Thu, 21 Nov 2024 19:38:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
51132024-11-21T19:38:18.178ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
51142024-11-21T19:38:18.179ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
51152024-11-21T19:38:18.179ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
51162024-11-21T19:38:18.180ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
51172024-11-21T19:38:18.180ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 348.271173ms iteration = 8 status = {"generation":2,"server_results":{"[fd00:1122:3344:101::8]:5353":{"Ok":null},"[fd00:1122:3344:101::9]:5353":{"Ok":null}}}
51182024-11-21T19:38:18.181ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
51192024-11-21T19:38:18.181ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
51202024-11-21T19:38:18.181ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
51212024-11-21T19:38:18.184ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 51 version = 1
51222024-11-21T19:38:18.184ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
51232024-11-21T19:38:18.184ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-11-21 19:33:31.797390 UTC dns_group = internal generation = 1
51242024-11-21T19:38:18.184ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 351.700019ms iteration = 6 status = {"generation":1}
51252024-11-21T19:38:18.189ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
51262024-11-21T19:38:18.189ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
51272024-11-21T19:38:18.193ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
51282024-11-21T19:38:18.193ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "7562d18a-1dba-4920-ae51-61ed4602fa15", "content-length": "1", "date": "Thu, 21 Nov 2024 19:38:18 GMT"} })
51292024-11-21T19:38:18.196ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
51302024-11-21T19:38:18.196ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
51312024-11-21T19:38:18.196ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 364.633092ms iteration = 11 status = {}
51322024-11-21T19:38:18.198ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 366.141612ms iteration = 11 status = null
51332024-11-21T19:38:18.209ZWARNnexus (BackgroundTasks): instance reincarnation completed with errors background_task = instance_reincarnation file = nexus/src/app/background/tasks/instance_reincarnation.rs:103 instances_changed_state = 0 instances_found = 0 instances_reincarnated = 0 query_errors = 0 restart_errors = 0
51342024-11-21T19:38:18.209ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_reincarnation elapsed = 376.754494ms iteration = 6 status = {"changed_state":[],"disabled":false,"errors":[],"instances_found":{"Failed":0,"SagaUnwound":0},"instances_reincarnated":[],"restart_errors":[]}
51352024-11-21T19:38:18.212ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
51362024-11-21T19:38:18.212ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
51372024-11-21T19:38:18.213ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
51382024-11-21T19:38:18.214ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
51392024-11-21T19:38:18.216ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
51402024-11-21T19:38:18.216ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
51412024-11-21T19:38:18.216ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
51422024-11-21T19:38:18.219ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000002, lookup_type: ByName("nexus") } result = Ok(())
51432024-11-21T19:38:18.220ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
51442024-11-21T19:38:18.220ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
51452024-11-21T19:38:18.248ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
51462024-11-21T19:38:18.248ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
51472024-11-21T19:38:18.249ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
51482024-11-21T19:38:18.251ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
51492024-11-21T19:38:18.251ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
51502024-11-21T19:38:18.251ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
51512024-11-21T19:38:18.251ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:1::1]:5353/config", status: 204, headers: {"x-request-id": "51969fd9-769d-4e89-bf26-f7a53546bd44", "date": "Thu, 21 Nov 2024 19:38:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
51522024-11-21T19:38:18.251ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
51532024-11-21T19:38:18.251ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
51542024-11-21T19:38:18.253ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:3::1]:5353/config", status: 204, headers: {"x-request-id": "3456f909-a2f5-4d4f-921b-aee926659c81", "date": "Thu, 21 Nov 2024 19:38:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
51552024-11-21T19:38:18.253ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
51562024-11-21T19:38:18.253ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
51572024-11-21T19:38:18.255ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:2::1]:5353/config", status: 204, headers: {"x-request-id": "8e1a3268-8412-4d76-b09b-bc81639337eb", "date": "Thu, 21 Nov 2024 19:38:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
51582024-11-21T19:38:18.255ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
51592024-11-21T19:38:18.255ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
51602024-11-21T19:38:18.255ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
51612024-11-21T19:38:18.255ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
51622024-11-21T19:38:18.257ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
51632024-11-21T19:38:18.257ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 424.664814ms iteration = 8 status = {"generation":1,"server_results":{"[fd00:1122:3344:1::1]:5353":{"Ok":null},"[fd00:1122:3344:2::1]:5353":{"Ok":null},"[fd00:1122:3344:3::1]:5353":{"Ok":null}}}
51642024-11-21T19:38:18.258ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-11-21T19:33:15.446964Z, time_modified: 2024-11-21T19:33:15.446964Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }) }
51652024-11-21T19:38:18.258ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 425.599867ms iteration = 6 status = {"by_dns_name":{"default-silo.sys.oxide.test":{"silo_id":"001de000-5110-4000-8000-000000000000","tls_certs":[]},"recovery.sys.oxide.test":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]}},"default_endpoint":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]},"warnings":["silo 001de000-5110-4000-8000-000000000000 with DNS name \\"default-silo.sys.oxide.test\\" has no usable certificates"]}
51662024-11-21T19:38:18.259ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
51672024-11-21T19:38:18.259ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
51682024-11-21T19:38:18.260ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
51692024-11-21T19:38:18.261ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
51702024-11-21T19:38:18.296ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
51712024-11-21T19:38:18.296ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "e5b7ec28-db87-481b-a22f-f4c85052192c", "date": "Thu, 21 Nov 2024 19:38:18 GMT"} })
51722024-11-21T19:38:18.296ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
51732024-11-21T19:38:18.296ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
51742024-11-21T19:38:18.298ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
51752024-11-21T19:38:18.299ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
51762024-11-21T19:38:18.301ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
51772024-11-21T19:38:18.301ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
51782024-11-21T19:38:18.302ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
51792024-11-21T19:38:18.302ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000002, lookup_type: ByName("nexus") } result = Ok(())
51802024-11-21T19:38:18.302ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
51812024-11-21T19:38:18.302ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
51822024-11-21T19:38:18.302ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
51832024-11-21T19:38:18.304ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
51842024-11-21T19:38:18.304ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
51852024-11-21T19:38:18.304ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
51862024-11-21T19:38:18.305ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
51872024-11-21T19:38:18.305ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
51882024-11-21T19:38:18.306ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "660948fe-f1d4-4f47-aaf8-cbe31eb81726", "date": "Thu, 21 Nov 2024 19:38:18 GMT"} })
51892024-11-21T19:38:18.307ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
51902024-11-21T19:38:18.307ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
51912024-11-21T19:38:18.307ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 474.882726ms iteration = 11 status = {}
51922024-11-21T19:38:18.313ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
51932024-11-21T19:38:18.313ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
51942024-11-21T19:38:18.315ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
51952024-11-21T19:38:18.316ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
51962024-11-21T19:38:18.316ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
51972024-11-21T19:38:18.316ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "3d917c2b-a8ce-4c0f-b2f3-9ea15ca9eb6f", "content-length": "134", "date": "Thu, 21 Nov 2024 19:38:18 GMT"} })
51982024-11-21T19:38:18.316ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
51992024-11-21T19:38:18.316ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
52002024-11-21T19:38:18.317ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
52012024-11-21T19:38:18.317ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
52022024-11-21T19:38:18.317ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
52032024-11-21T19:38:18.317ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
52042024-11-21T19:38:18.317ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
52052024-11-21T19:38:18.317ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
52062024-11-21T19:38:18.317ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
52072024-11-21T19:38:18.317ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
52082024-11-21T19:38:18.317ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
52092024-11-21T19:38:18.317ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
52102024-11-21T19:38:18.317ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
52112024-11-21T19:38:18.319ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
52122024-11-21T19:38:18.319ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
52132024-11-21T19:38:18.319ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
52142024-11-21T19:38:18.319ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "c6e28c8d-39bd-41ce-9324-55bfd765d082", "content-length": "2", "date": "Thu, 21 Nov 2024 19:38:18 GMT"} })
52152024-11-21T19:38:18.319ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
52162024-11-21T19:38:18.319ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "56787d72-ae3f-4c77-913a-cf5c53ae4c8c", "content-length": "51", "date": "Thu, 21 Nov 2024 19:38:18 GMT"} })
52172024-11-21T19:38:18.320ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
52182024-11-21T19:38:18.320ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
52192024-11-21T19:38:18.321ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
52202024-11-21T19:38:18.321ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
52212024-11-21T19:38:18.322ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
52222024-11-21T19:38:18.323ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
52232024-11-21T19:38:18.324ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
52242024-11-21T19:38:18.324ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
52252024-11-21T19:38:18.326ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
52262024-11-21T19:38:18.329ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000001, lookup_type: ByName("external-dns") } result = Ok(())
52272024-11-21T19:38:18.329ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
52282024-11-21T19:38:18.330ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
52292024-11-21T19:38:18.331ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
52302024-11-21T19:38:18.331ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
52312024-11-21T19:38:18.332ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
52322024-11-21T19:38:18.333ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
52332024-11-21T19:38:18.354ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
52342024-11-21T19:38:18.355ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
52352024-11-21T19:38:18.358ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "8b32aaa0-da09-43ac-8483-e44150ef6082", "content-length": "691", "date": "Thu, 21 Nov 2024 19:38:18 GMT"} })
52362024-11-21T19:38:18.358ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
52372024-11-21T19:38:18.358ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
52382024-11-21T19:38:18.358ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
52392024-11-21T19:38:18.359ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000001, lookup_type: ByName("external-dns") } result = Ok(())
52402024-11-21T19:38:18.359ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
52412024-11-21T19:38:18.359ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
52422024-11-21T19:38:18.360ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 527.805662ms iteration = 11 status = {}
52432024-11-21T19:38:18.368ZDEBGnexus (service-firewall-rule-progator): resolved names for firewall rules background_task = service_firewall_rule_propagation instance_interfaces = {} subnet_interfaces = {Name("external-dns"): [NetworkInterface { id: 4c211da9-c36c-41bc-8274-9aa6420c20e6, kind: Service { id: 90b200e8-d1ee-4e0d-975c-64628c5532d2 }, name: Name("external-dns-90b200e8-d1ee-4e0d-975c-64628c5532d2"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 185])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b69bf686-c443-40c6-b854-c2ff6e5e107a, kind: Service { id: 4e669ff4-6dde-4cd2-8491-623460b56d8d }, name: Name("external-dns-4e669ff4-6dde-4cd2-8491-623460b56d8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 188])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], Name("nexus"): [NetworkInterface { id: 03ccecdf-44ea-4815-8920-5ebe8d7bf7ff, kind: Service { id: 915e11b8-d7cb-4730-9906-7980540813dc }, name: Name("nexus-915e11b8-d7cb-4730-9906-7980540813dc"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 146])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 4cb2fda6-91fb-43d0-be7c-77f85347c432, kind: Service { id: 39d14e41-6d5d-474a-aaea-65c36b8f9da5 }, name: Name("nexus-39d14e41-6d5d-474a-aaea-65c36b8f9da5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 243])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7e0b7555-efc0-41d1-b3ba-7ca41e2e30b7, kind: Service { id: 088dbd0f-363c-4785-9d04-aa968256294e }, name: Name("nexus-088dbd0f-363c-4785-9d04-aa968256294e"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 205])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }]} subnet_networks = {Name("external-dns"): [V4(Ipv4Network { addr: 172.30.1.0, prefix: 24 }), V6(Ipv6Network { addr: fd77:e9d2:9cd9:1::, prefix: 64 })], Name("nexus"): [V4(Ipv4Network { addr: 172.30.2.0, prefix: 24 }), V6(Ipv6Network { addr: fd77:e9d2:9cd9:2::, prefix: 64 })]} vpc_interfaces = {}
52442024-11-21T19:38:18.368ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
52452024-11-21T19:38:18.368ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
52462024-11-21T19:38:18.369ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
52472024-11-21T19:38:18.371ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Fleet result = Ok(())
52482024-11-21T19:38:18.371ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
52492024-11-21T19:38:18.371ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
52502024-11-21T19:38:18.373ZDEBGnexus (service-firewall-rule-progator): Allowlist for user-facing services is set to allow any inbound traffic. Existing VPC firewall rules will not be modified. background_task = service_firewall_rule_propagation
52512024-11-21T19:38:18.373ZDEBGnexus (service-firewall-rule-progator): resolved firewall rules for sled agents background_task = service_firewall_rule_propagation sled_agent_rules = [ResolvedVpcFirewallRule { action: Allow, direction: Inbound, filter_hosts: None, filter_ports: Some([L4PortRange("53")]), filter_protocols: Some([Udp]), priority: 65534, status: Enabled, targets: [NetworkInterface { id: 4c211da9-c36c-41bc-8274-9aa6420c20e6, kind: Service { id: 90b200e8-d1ee-4e0d-975c-64628c5532d2 }, name: Name("external-dns-90b200e8-d1ee-4e0d-975c-64628c5532d2"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 185])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b69bf686-c443-40c6-b854-c2ff6e5e107a, kind: Service { id: 4e669ff4-6dde-4cd2-8491-623460b56d8d }, name: Name("external-dns-4e669ff4-6dde-4cd2-8491-623460b56d8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 188])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }] }, ResolvedVpcFirewallRule { action: Allow, direction: Inbound, filter_hosts: None, filter_ports: Some([L4PortRange("80"), L4PortRange("443")]), filter_protocols: Some([Tcp]), priority: 65534, status: Enabled, targets: [NetworkInterface { id: 03ccecdf-44ea-4815-8920-5ebe8d7bf7ff, kind: Service { id: 915e11b8-d7cb-4730-9906-7980540813dc }, name: Name("nexus-915e11b8-d7cb-4730-9906-7980540813dc"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 146])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 4cb2fda6-91fb-43d0-be7c-77f85347c432, kind: Service { id: 39d14e41-6d5d-474a-aaea-65c36b8f9da5 }, name: Name("nexus-39d14e41-6d5d-474a-aaea-65c36b8f9da5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 243])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7e0b7555-efc0-41d1-b3ba-7ca41e2e30b7, kind: Service { id: 088dbd0f-363c-4785-9d04-aa968256294e }, name: Name("nexus-088dbd0f-363c-4785-9d04-aa968256294e"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 205])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }] }]
52522024-11-21T19:38:18.373ZDEBGnexus (service-firewall-rule-progator): resolved 2 rules for sleds background_task = service_firewall_rule_propagation
52532024-11-21T19:38:18.431ZDEBGnexus (service-firewall-rule-progator): resolved sleds for vpc oxide-services background_task = service_firewall_rule_propagation vpc_to_sled = [Sled { identity: SledIdentity { id: 203795ed-576b-40e4-bed8-7463e3f2162c, time_created: 2024-11-21T19:33:30.261358Z, time_modified: 2024-11-21T19:33:30.261358Z }, time_deleted: None, rcgen: Generation(Generation(19)), rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, is_scrimlet: true, serial_number: "buskin", part_number: "i86pc", revision: SqlU32(0), usable_hardware_threads: SqlU32(48), usable_physical_ram: ByteCount(ByteCount(102952026112)), reservoir_size: ByteCount(ByteCount(51474595840)), ip: fd00:1122:3344:101::1, port: SqlU16(12345), last_used_address: fd00:1122:3344:101::ffff, policy: InService, state: Active, sled_agent_gen: Generation(Generation(1)) }]
52542024-11-21T19:38:18.431ZDEBGnexus (service-firewall-rule-progator): sending firewall rules to sled agents background_task = service_firewall_rule_propagation
52552024-11-21T19:38:18.433ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
52562024-11-21T19:38:18.434ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
52572024-11-21T19:38:18.436ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
52582024-11-21T19:38:18.436ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
52592024-11-21T19:38:18.438ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
52602024-11-21T19:38:18.440ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
52612024-11-21T19:38:18.471ZDEBGnexus (service-firewall-rule-progator): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = service_firewall_rule_propagation body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc/001de000-074c-4000-8000-000000000000/firewall/rules
52622024-11-21T19:38:18.475ZDEBGnexus (service-firewall-rule-progator): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = service_firewall_rule_propagation result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc/001de000-074c-4000-8000-000000000000/firewall/rules", status: 204, headers: {"x-request-id": "6be5bbbe-9496-4261-a5f8-e0ef77fe168f", "date": "Thu, 21 Nov 2024 19:38:18 GMT"} })
52632024-11-21T19:38:18.476ZINFOnexus (service-firewall-rule-progator): updated firewall rules on 1 sleds background_task = service_firewall_rule_propagation file = nexus/networking/src/firewall_rules.rs:496
52642024-11-21T19:38:18.476ZDEBGnexus (service-firewall-rule-progator): successfully propagated service firewall rules background_task = service_firewall_rule_propagation elapsed = 643.60885ms
52652024-11-21T19:38:18.476ZDEBGnexus (BackgroundTasks): activation complete background_task = service_firewall_rule_propagation elapsed = 643.61767ms iteration = 2 status = {}
52662024-11-21T19:38:18.504ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "928d5c9e-30bd-4fe7-8a1f-67b1405de9f2", "date": "Thu, 21 Nov 2024 19:38:18 GMT"} })
52672024-11-21T19:38:18.504ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
52682024-11-21T19:38:18.504ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
52692024-11-21T19:38:18.504ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "f40791ce-afe3-4063-8e18-1d19e82069be", "content-length": "682", "date": "Thu, 21 Nov 2024 19:38:18 GMT"} })
52702024-11-21T19:38:18.505ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
52712024-11-21T19:38:18.505ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
52722024-11-21T19:38:18.507ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
52732024-11-21T19:38:18.508ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
52742024-11-21T19:38:18.511ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
52752024-11-21T19:38:18.511ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
52762024-11-21T19:38:18.513ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
52772024-11-21T19:38:18.513ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
52782024-11-21T19:38:18.521ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 688.934185ms iteration = 11 status = {}
52792024-11-21T19:38:26.961ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 311 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = bdf0432e-6f3b-4257-b191-f880d2f5b480 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
52802024-11-21T19:38:27.831ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 32 reason = Timeout
52812024-11-21T19:38:27.831ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
52822024-11-21T19:38:27.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
52832024-11-21T19:38:27.833ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 32 reason = Timeout
52842024-11-21T19:38:27.833ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
52852024-11-21T19:38:27.833ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
52862024-11-21T19:38:27.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
52872024-11-21T19:38:27.836ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
52882024-11-21T19:38:27.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
52892024-11-21T19:38:27.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
52902024-11-21T19:38:27.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
52912024-11-21T19:38:27.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
52922024-11-21T19:38:27.839ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
52932024-11-21T19:38:27.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
52942024-11-21T19:38:27.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
52952024-11-21T19:38:27.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
52962024-11-21T19:38:27.851ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
52972024-11-21T19:38:27.851ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 19.139406ms iteration = 32 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
52982024-11-21T19:38:27.862ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
52992024-11-21T19:38:27.862ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
53002024-11-21T19:38:27.864ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 30.311625ms iteration = 32 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
53012024-11-21T19:38:37.832ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 33 reason = Timeout
53022024-11-21T19:38:37.832ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
53032024-11-21T19:38:37.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
53042024-11-21T19:38:37.832ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 33 reason = Timeout
53052024-11-21T19:38:37.832ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
53062024-11-21T19:38:37.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
53072024-11-21T19:38:37.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
53082024-11-21T19:38:37.836ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
53092024-11-21T19:38:37.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
53102024-11-21T19:38:37.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
53112024-11-21T19:38:37.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
53122024-11-21T19:38:37.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
53132024-11-21T19:38:37.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
53142024-11-21T19:38:37.843ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
53152024-11-21T19:38:37.843ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
53162024-11-21T19:38:37.843ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
53172024-11-21T19:38:37.852ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
53182024-11-21T19:38:37.852ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 20.505984ms iteration = 33 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
53192024-11-21T19:38:37.860ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
53202024-11-21T19:38:37.861ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
53212024-11-21T19:38:37.862ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 29.503034ms iteration = 33 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
53222024-11-21T19:38:41.960ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 319 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 935eb322-5f22-4541-9b1e-34009cf866fa response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
53232024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 12 reason = Timeout
53242024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
53252024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 34 reason = Timeout
53262024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 13 reason = Timeout
53272024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
53282024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
53292024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 12 reason = Timeout
53302024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 12 reason = Timeout
53312024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
53322024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
53332024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 12 reason = Timeout
53342024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 12 reason = Timeout
53352024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
53362024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
53372024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 12 reason = Timeout
53382024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
53392024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 12 reason = Timeout
53402024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
53412024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
53422024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
53432024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
53442024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
53452024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
53462024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
53472024-11-21T19:38:47.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
53482024-11-21T19:38:47.833ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 12 reason = Timeout
53492024-11-21T19:38:47.833ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
53502024-11-21T19:38:47.833ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 12 reason = Timeout
53512024-11-21T19:38:47.833ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
53522024-11-21T19:38:47.833ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
53532024-11-21T19:38:47.833ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 12 reason = Timeout
53542024-11-21T19:38:47.833ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
53552024-11-21T19:38:47.833ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 12 reason = Timeout
53562024-11-21T19:38:47.833ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
53572024-11-21T19:38:47.833ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 34 reason = Timeout
53582024-11-21T19:38:47.833ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
53592024-11-21T19:38:47.833ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 12 reason = Timeout
53602024-11-21T19:38:47.833ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
53612024-11-21T19:38:47.833ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
53622024-11-21T19:38:47.833ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 12 reason = Timeout
53632024-11-21T19:38:47.833ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
53642024-11-21T19:38:47.833ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 12 reason = Timeout
53652024-11-21T19:38:47.833ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
53662024-11-21T19:38:47.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
53672024-11-21T19:38:47.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
53682024-11-21T19:38:47.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
53692024-11-21T19:38:47.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
53702024-11-21T19:38:47.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
53712024-11-21T19:38:47.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
53722024-11-21T19:38:47.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
53732024-11-21T19:38:47.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
53742024-11-21T19:38:47.835ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1070, tv_nsec: 711508600 } })
53752024-11-21T19:38:47.835ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
53762024-11-21T19:38:47.835ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
53772024-11-21T19:38:47.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
53782024-11-21T19:38:47.837ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
53792024-11-21T19:38:47.837ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 5.006787ms iteration = 12 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
53802024-11-21T19:38:47.838ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1070, tv_nsec: 714144945 } })
53812024-11-21T19:38:47.838ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
53822024-11-21T19:38:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
53832024-11-21T19:38:47.839ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
53842024-11-21T19:38:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
53852024-11-21T19:38:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
53862024-11-21T19:38:47.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
53872024-11-21T19:38:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
53882024-11-21T19:38:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
53892024-11-21T19:38:47.840ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
53902024-11-21T19:38:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
53912024-11-21T19:38:47.842ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
53922024-11-21T19:38:47.843ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
53932024-11-21T19:38:47.843ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
53942024-11-21T19:38:47.843ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
53952024-11-21T19:38:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
53962024-11-21T19:38:47.845ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
53972024-11-21T19:38:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
53982024-11-21T19:38:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
53992024-11-21T19:38:47.845ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
54002024-11-21T19:38:47.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
54012024-11-21T19:38:47.847ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
54022024-11-21T19:38:47.847ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
54032024-11-21T19:38:47.847ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
54042024-11-21T19:38:47.849ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
54052024-11-21T19:38:47.849ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
54062024-11-21T19:38:47.849ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
54072024-11-21T19:38:47.849ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
54082024-11-21T19:38:47.849ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
54092024-11-21T19:38:47.849ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
54102024-11-21T19:38:47.850ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
54112024-11-21T19:38:47.851ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1070, tv_nsec: 727468383 } })
54122024-11-21T19:38:47.851ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
54132024-11-21T19:38:47.852ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
54142024-11-21T19:38:47.852ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
54152024-11-21T19:38:47.864ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
54162024-11-21T19:38:47.865ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
54172024-11-21T19:38:47.885ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
54182024-11-21T19:38:47.886ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
54192024-11-21T19:38:47.886ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 52.872787ms iteration = 12 status = {"errors":[],"records_set_to_done":[]}
54202024-11-21T19:38:47.888ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
54212024-11-21T19:38:47.888ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
54222024-11-21T19:38:47.889ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
54232024-11-21T19:38:47.890ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
54242024-11-21T19:38:47.890ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
54252024-11-21T19:38:47.890ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 56.90031ms iteration = 12 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
54262024-11-21T19:38:47.890ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
54272024-11-21T19:38:47.891ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
54282024-11-21T19:38:47.892ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
54292024-11-21T19:38:47.892ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 60.414828ms iteration = 12 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
54302024-11-21T19:38:47.893ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
54312024-11-21T19:38:47.893ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
54322024-11-21T19:38:47.893ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
54332024-11-21T19:38:47.893ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
54342024-11-21T19:38:47.893ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
54352024-11-21T19:38:47.893ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
54362024-11-21T19:38:47.894ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
54372024-11-21T19:38:47.895ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 61.886257ms iteration = 12 status = {"errors":[],"garbage_collect_requested":[]}
54382024-11-21T19:38:47.896ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
54392024-11-21T19:38:47.897ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
54402024-11-21T19:38:47.898ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
54412024-11-21T19:38:47.898ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
54422024-11-21T19:38:47.899ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
54432024-11-21T19:38:47.899ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
54442024-11-21T19:38:47.900ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
54452024-11-21T19:38:47.900ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
54462024-11-21T19:38:47.900ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
54472024-11-21T19:38:47.900ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 68.432534ms iteration = 12 status = 0
54482024-11-21T19:38:47.900ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
54492024-11-21T19:38:47.901ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
54502024-11-21T19:38:47.901ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
54512024-11-21T19:38:47.901ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
54522024-11-21T19:38:47.901ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 67.763826ms iteration = 12 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
54532024-11-21T19:38:47.901ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 67.812756ms iteration = 34 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
54542024-11-21T19:38:47.902ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
54552024-11-21T19:38:47.902ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
54562024-11-21T19:38:47.903ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
54572024-11-21T19:38:47.908ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
54582024-11-21T19:38:47.910ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
54592024-11-21T19:38:47.910ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
54602024-11-21T19:38:47.910ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
54612024-11-21T19:38:47.911ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
54622024-11-21T19:38:47.913ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
54632024-11-21T19:38:47.913ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
54642024-11-21T19:38:47.913ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
54652024-11-21T19:38:47.915ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
54662024-11-21T19:38:47.915ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 83.320893ms iteration = 34 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
54672024-11-21T19:38:47.922ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 88.660724ms iteration = 12 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
54682024-11-21T19:38:47.925ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
54692024-11-21T19:38:47.925ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
54702024-11-21T19:38:47.925ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
54712024-11-21T19:38:47.925ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
54722024-11-21T19:38:47.934ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
54732024-11-21T19:38:47.934ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
54742024-11-21T19:38:47.934ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
54752024-11-21T19:38:47.934ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
54762024-11-21T19:38:47.934ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "728c4ae2-b733-48e7-b0df-8e8e2936cb34", "content-length": "26", "date": "Thu, 21 Nov 2024 19:38:47 GMT"} })
54772024-11-21T19:38:47.934ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
54782024-11-21T19:38:47.934ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "543066f7-e38b-4414-97e7-844ecadc4950", "content-length": "26", "date": "Thu, 21 Nov 2024 19:38:47 GMT"} })
54792024-11-21T19:38:47.934ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
54802024-11-21T19:38:47.935ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
54812024-11-21T19:38:47.936ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
54822024-11-21T19:38:47.940ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 107.72753ms iteration = 13 status = {"physical_disks_added":0}
54832024-11-21T19:38:47.955ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
54842024-11-21T19:38:47.955ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
54852024-11-21T19:38:47.955ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 121.401892ms iteration = 12 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
54862024-11-21T19:38:47.957ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
54872024-11-21T19:38:47.957ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
54882024-11-21T19:38:47.957ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "8c9c9167-ced7-4106-87c2-1aec1ba9aafd", "content-length": "26", "date": "Thu, 21 Nov 2024 19:38:47 GMT"} })
54892024-11-21T19:38:47.957ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
54902024-11-21T19:38:47.958ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
54912024-11-21T19:38:48.003ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
54922024-11-21T19:38:48.003ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
54932024-11-21T19:38:48.010ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "4419e36b-a9b3-49a6-8fc3-b63d0733c26a", "content-length": "372", "date": "Thu, 21 Nov 2024 19:38:47 GMT"} })
54942024-11-21T19:38:48.010ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
54952024-11-21T19:38:48.010ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
54962024-11-21T19:38:48.029ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 196.793419ms iteration = 12 status = {}
54972024-11-21T19:38:48.030ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
54982024-11-21T19:38:48.030ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "a2af59b3-1807-4d77-a496-435276572d19", "content-length": "1", "date": "Thu, 21 Nov 2024 19:38:47 GMT"} })
54992024-11-21T19:38:48.031ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
55002024-11-21T19:38:48.031ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
55012024-11-21T19:38:48.052ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
55022024-11-21T19:38:48.053ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "2102f6c2-066a-469c-b53b-9d646e2220d5", "content-length": "691", "date": "Thu, 21 Nov 2024 19:38:47 GMT"} })
55032024-11-21T19:38:48.053ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
55042024-11-21T19:38:48.053ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
55052024-11-21T19:38:48.053ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
55062024-11-21T19:38:48.055ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 222.620864ms iteration = 12 status = {}
55072024-11-21T19:38:48.055ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 222.757246ms iteration = 12 status = null
55082024-11-21T19:38:48.055ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
55092024-11-21T19:38:48.055ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
55102024-11-21T19:38:48.058ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "d0f049fe-b150-43ae-b945-3a1b8099c48d", "date": "Thu, 21 Nov 2024 19:38:47 GMT"} })
55112024-11-21T19:38:48.058ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
55122024-11-21T19:38:48.058ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
55132024-11-21T19:38:48.060ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
55142024-11-21T19:38:48.060ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
55152024-11-21T19:38:48.063ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
55162024-11-21T19:38:48.063ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
55172024-11-21T19:38:48.066ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
55182024-11-21T19:38:48.066ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
55192024-11-21T19:38:48.067ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "17737a7f-9e34-40d2-bd65-6003cb458f43", "date": "Thu, 21 Nov 2024 19:38:47 GMT"} })
55202024-11-21T19:38:48.068ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 234.665386ms iteration = 12 status = {}
55212024-11-21T19:38:48.110ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
55222024-11-21T19:38:48.110ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
55232024-11-21T19:38:48.111ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
55242024-11-21T19:38:48.111ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
55252024-11-21T19:38:48.111ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
55262024-11-21T19:38:48.120ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
55272024-11-21T19:38:48.120ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "d90afe17-0464-4b45-b5f8-ad955211ef36", "content-length": "134", "date": "Thu, 21 Nov 2024 19:38:47 GMT"} })
55282024-11-21T19:38:48.120ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
55292024-11-21T19:38:48.120ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
55302024-11-21T19:38:48.120ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
55312024-11-21T19:38:48.120ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
55322024-11-21T19:38:48.120ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
55332024-11-21T19:38:48.120ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
55342024-11-21T19:38:48.120ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
55352024-11-21T19:38:48.120ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
55362024-11-21T19:38:48.120ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
55372024-11-21T19:38:48.120ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
55382024-11-21T19:38:48.121ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
55392024-11-21T19:38:48.121ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
55402024-11-21T19:38:48.121ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
55412024-11-21T19:38:48.122ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
55422024-11-21T19:38:48.122ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "22a1b3c3-efe7-4a11-91c5-2abd11d42817", "content-length": "2", "date": "Thu, 21 Nov 2024 19:38:47 GMT"} })
55432024-11-21T19:38:48.122ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
55442024-11-21T19:38:48.122ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "ba9b8fda-8fa7-422d-9c03-cb0b14327445", "content-length": "51", "date": "Thu, 21 Nov 2024 19:38:47 GMT"} })
55452024-11-21T19:38:48.122ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
55462024-11-21T19:38:48.122ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
55472024-11-21T19:38:48.216ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "8e72efc4-93be-4732-9406-845c03949a9e", "date": "Thu, 21 Nov 2024 19:38:47 GMT"} })
55482024-11-21T19:38:48.216ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
55492024-11-21T19:38:48.216ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
55502024-11-21T19:38:48.216ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "7e6bd75f-6194-441d-bbe2-4988b59869e0", "content-length": "682", "date": "Thu, 21 Nov 2024 19:38:47 GMT"} })
55512024-11-21T19:38:48.216ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
55522024-11-21T19:38:48.217ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
55532024-11-21T19:38:48.221ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
55542024-11-21T19:38:48.221ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
55552024-11-21T19:38:48.225ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
55562024-11-21T19:38:48.225ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
55572024-11-21T19:38:48.227ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
55582024-11-21T19:38:48.227ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
55592024-11-21T19:38:48.234ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 402.220414ms iteration = 12 status = {}
55602024-11-21T19:38:56.961ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 278 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 757fd781-e9b1-4fb9-965a-8244dfa07459 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
55612024-11-21T19:38:57.832ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 35 reason = Timeout
55622024-11-21T19:38:57.832ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
55632024-11-21T19:38:57.832ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
55642024-11-21T19:38:57.833ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 35 reason = Timeout
55652024-11-21T19:38:57.833ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
55662024-11-21T19:38:57.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
55672024-11-21T19:38:57.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
55682024-11-21T19:38:57.837ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
55692024-11-21T19:38:57.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
55702024-11-21T19:38:57.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
55712024-11-21T19:38:57.839ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
55722024-11-21T19:38:57.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
55732024-11-21T19:38:57.844ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
55742024-11-21T19:38:57.847ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
55752024-11-21T19:38:57.847ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
55762024-11-21T19:38:57.847ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
55772024-11-21T19:38:57.860ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
55782024-11-21T19:38:57.860ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 28.028935ms iteration = 35 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
55792024-11-21T19:38:57.868ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
55802024-11-21T19:38:57.868ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
55812024-11-21T19:38:57.869ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 36.290785ms iteration = 35 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
55822024-11-21T19:39:07.832ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 36 reason = Timeout
55832024-11-21T19:39:07.832ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
55842024-11-21T19:39:07.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
55852024-11-21T19:39:07.833ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 36 reason = Timeout
55862024-11-21T19:39:07.833ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
55872024-11-21T19:39:07.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
55882024-11-21T19:39:07.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
55892024-11-21T19:39:07.837ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
55902024-11-21T19:39:07.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
55912024-11-21T19:39:07.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
55922024-11-21T19:39:07.839ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
55932024-11-21T19:39:07.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
55942024-11-21T19:39:07.841ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
55952024-11-21T19:39:07.843ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
55962024-11-21T19:39:07.843ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
55972024-11-21T19:39:07.844ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
55982024-11-21T19:39:07.855ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
55992024-11-21T19:39:07.855ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 22.843515ms iteration = 36 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
56002024-11-21T19:39:07.862ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
56012024-11-21T19:39:07.862ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
56022024-11-21T19:39:07.863ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 29.751737ms iteration = 36 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
56032024-11-21T19:39:11.961ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 311 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 4e509c27-74f4-4d66-8349-071be2398726 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
56042024-11-21T19:39:17.832ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 9 reason = Timeout
56052024-11-21T19:39:17.832ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 13 reason = Timeout
56062024-11-21T19:39:17.832ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
56072024-11-21T19:39:17.832ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 13 reason = Timeout
56082024-11-21T19:39:17.832ZDEBGnexus (BackgroundTasks): activating background_task = crdb_node_id_collector iteration = 4 reason = Timeout
56092024-11-21T19:39:17.832ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
56102024-11-21T19:39:17.832ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 8 reason = Timeout
56112024-11-21T19:39:17.832ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:83 reason = blueprint disabled target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb
56122024-11-21T19:39:17.832ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
56132024-11-21T19:39:17.832ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 13 reason = Timeout
56142024-11-21T19:39:17.832ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 19.57\u{b5}s iteration = 8 status = {"enabled":false,"target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb"}
56152024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 7 reason = Timeout
56162024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 7 reason = Timeout
56172024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
56182024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 13 reason = Timeout
56192024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
56202024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
56212024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 9 reason = Timeout
56222024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 14 reason = Timeout
56232024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
56242024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 37 reason = Timeout
56252024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 13 reason = Timeout
56262024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
56272024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
56282024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 7 reason = Timeout
56292024-11-21T19:39:17.833ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
56302024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 7 reason = Timeout
56312024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 7 reason = Timeout
56322024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
56332024-11-21T19:39:17.833ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-11-21 19:29:17.833034267 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
56342024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
56352024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
56362024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
56372024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 13 reason = Timeout
56382024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 7 reason = Timeout
56392024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 13 reason = Timeout
56402024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 7 reason = Timeout
56412024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
56422024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
56432024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
56442024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
56452024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
56462024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
56472024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
56482024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
56492024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
56502024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
56512024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
56522024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
56532024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
56542024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
56552024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
56562024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
56572024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
56582024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
56592024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 37 reason = Timeout
56602024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 13 reason = Timeout
56612024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = instance_reincarnation iteration = 7 reason = Timeout
56622024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
56632024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
56642024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 7 reason = Timeout
56652024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
56662024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
56672024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
56682024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 13 reason = Timeout
56692024-11-21T19:39:17.833ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
56702024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
56712024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
56722024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 13 reason = Timeout
56732024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
56742024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 7 reason = Timeout
56752024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
56762024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 13 reason = Timeout
56772024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 13 reason = Timeout
56782024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
56792024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
56802024-11-21T19:39:17.833ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 13 reason = Timeout
56812024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
56822024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 13 reason = Timeout
56832024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
56842024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
56852024-11-21T19:39:17.834ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0377cf96-baf3-4e09-a570-eb7c67037d4d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0d1b3b31-4591-4bcb-931d-e8e03365f0d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:1::1))) }], valid_until: Instant { tv_sec: 1100, tv_nsec: 709580754 } })
56862024-11-21T19:39:17.834ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("4e669ff4-6dde-4cd2-8491-623460b56d8d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 1100, tv_nsec: 709612285 } })
56872024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
56882024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
56892024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
56902024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
56912024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
56922024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
56932024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
56942024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
56952024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
56962024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
56972024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
56982024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
56992024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
57002024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
57012024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
57022024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
57032024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } dns_group = internal
57042024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 1.566941ms iteration = 7 status = {"addresses":["[fd00:1122:3344:3::1]:5353","[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353"]}
57052024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
57062024-11-21T19:39:17.834ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 1.792944ms iteration = 7 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
57072024-11-21T19:39:17.840ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
57082024-11-21T19:39:17.841ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1100, tv_nsec: 716957543 } })
57092024-11-21T19:39:17.841ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 8.335421ms iteration = 13 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
57102024-11-21T19:39:17.841ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
57112024-11-21T19:39:17.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
57122024-11-21T19:39:17.842ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
57132024-11-21T19:39:17.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
57142024-11-21T19:39:17.843ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1100, tv_nsec: 718906789 } })
57152024-11-21T19:39:17.843ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
57162024-11-21T19:39:17.844ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
57172024-11-21T19:39:17.844ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
57182024-11-21T19:39:17.844ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
57192024-11-21T19:39:17.845ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-11-21 19:29:17.833034267 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
57202024-11-21T19:39:17.845ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 12.127011ms iteration = 7 status = {"expiration":"2024-11-21T19:29:17.833034267Z","pruned":[]}
57212024-11-21T19:39:17.845ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
57222024-11-21T19:39:17.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
57232024-11-21T19:39:17.847ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
57242024-11-21T19:39:17.847ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
57252024-11-21T19:39:17.847ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
57262024-11-21T19:39:17.849ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
57272024-11-21T19:39:17.850ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
57282024-11-21T19:39:17.853ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
57292024-11-21T19:39:17.854ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
57302024-11-21T19:39:17.854ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
57312024-11-21T19:39:17.854ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
57322024-11-21T19:39:17.855ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
57332024-11-21T19:39:17.855ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
57342024-11-21T19:39:17.855ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
57352024-11-21T19:39:17.856ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
57362024-11-21T19:39:17.856ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
57372024-11-21T19:39:17.856ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
57382024-11-21T19:39:17.856ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
57392024-11-21T19:39:17.856ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
57402024-11-21T19:39:17.864ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
57412024-11-21T19:39:17.865ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 31.328279ms iteration = 7 status = {"errors":[],"found_port_ok":[]}
57422024-11-21T19:39:17.868ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
57432024-11-21T19:39:17.868ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
57442024-11-21T19:39:17.868ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
57452024-11-21T19:39:17.873ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
57462024-11-21T19:39:17.878ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
57472024-11-21T19:39:17.887ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
57482024-11-21T19:39:17.892ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
57492024-11-21T19:39:17.903ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
57502024-11-21T19:39:17.903ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
57512024-11-21T19:39:17.905ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
57522024-11-21T19:39:17.906ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
57532024-11-21T19:39:17.907ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
57542024-11-21T19:39:17.910ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
57552024-11-21T19:39:17.910ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
57562024-11-21T19:39:17.910ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
57572024-11-21T19:39:17.912ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
57582024-11-21T19:39:17.912ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
57592024-11-21T19:39:17.912ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
57602024-11-21T19:39:17.912ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
57612024-11-21T19:39:17.912ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
57622024-11-21T19:39:17.913ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
57632024-11-21T19:39:17.916ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
57642024-11-21T19:39:17.916ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
57652024-11-21T19:39:17.916ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
57662024-11-21T19:39:17.917ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
57672024-11-21T19:39:17.917ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1100, tv_nsec: 793056370 } })
57682024-11-21T19:39:17.917ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
57692024-11-21T19:39:17.917ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 84.120615ms iteration = 13 status = {"errors":[],"records_set_to_done":[]}
57702024-11-21T19:39:17.918ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
57712024-11-21T19:39:17.919ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
57722024-11-21T19:39:17.919ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
57732024-11-21T19:39:17.920ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
57742024-11-21T19:39:17.920ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
57752024-11-21T19:39:17.920ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
57762024-11-21T19:39:17.920ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
57772024-11-21T19:39:17.920ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
57782024-11-21T19:39:17.920ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
57792024-11-21T19:39:17.921ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
57802024-11-21T19:39:17.922ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
57812024-11-21T19:39:17.922ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
57822024-11-21T19:39:17.922ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
57832024-11-21T19:39:17.922ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 88.845877ms iteration = 13 status = {"errors":[],"garbage_collect_requested":[]}
57842024-11-21T19:39:17.922ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
57852024-11-21T19:39:17.923ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
57862024-11-21T19:39:17.923ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
57872024-11-21T19:39:17.924ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
57882024-11-21T19:39:17.925ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
57892024-11-21T19:39:17.925ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
57902024-11-21T19:39:17.925ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
57912024-11-21T19:39:17.926ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0 total = 0
57922024-11-21T19:39:17.926ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:171 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0 vmms_found = 0
57932024-11-21T19:39:17.926ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 92.921731ms iteration = 7 status = {"errors":[],"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0,"vmms_found":0}
57942024-11-21T19:39:17.927ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
57952024-11-21T19:39:17.927ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
57962024-11-21T19:39:17.927ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
57972024-11-21T19:39:17.928ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 95.175551ms iteration = 13 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
57982024-11-21T19:39:17.929ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
57992024-11-21T19:39:17.930ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
58002024-11-21T19:39:17.930ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
58012024-11-21T19:39:17.931ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
58022024-11-21T19:39:17.931ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
58032024-11-21T19:39:17.931ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
58042024-11-21T19:39:17.931ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
58052024-11-21T19:39:17.931ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
58062024-11-21T19:39:17.931ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
58072024-11-21T19:39:17.934ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
58082024-11-21T19:39:17.934ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
58092024-11-21T19:39:17.935ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
58102024-11-21T19:39:17.935ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
58112024-11-21T19:39:17.936ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
58122024-11-21T19:39:17.936ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
58132024-11-21T19:39:17.940ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
58142024-11-21T19:39:17.942ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
58152024-11-21T19:39:17.944ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
58162024-11-21T19:39:17.944ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
58172024-11-21T19:39:17.944ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
58182024-11-21T19:39:17.945ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
58192024-11-21T19:39:17.945ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
58202024-11-21T19:39:17.945ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
58212024-11-21T19:39:17.945ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
58222024-11-21T19:39:17.946ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
58232024-11-21T19:39:17.946ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
58242024-11-21T19:39:17.946ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
58252024-11-21T19:39:17.946ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
58262024-11-21T19:39:17.950ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
58272024-11-21T19:39:17.950ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
58282024-11-21T19:39:17.950ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
58292024-11-21T19:39:17.950ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
58302024-11-21T19:39:17.950ZDEBGnexus (BackgroundTasks): activation complete background_task = crdb_node_id_collector elapsed = 118.095358ms iteration = 4 status = {"nsuccess":5}
58312024-11-21T19:39:17.951ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
58322024-11-21T19:39:17.951ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
58332024-11-21T19:39:17.953ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
58342024-11-21T19:39:17.955ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
58352024-11-21T19:39:17.955ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
58362024-11-21T19:39:17.955ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 121.98867ms iteration = 7 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
58372024-11-21T19:39:17.957ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
58382024-11-21T19:39:17.957ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
58392024-11-21T19:39:17.957ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
58402024-11-21T19:39:17.958ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
58412024-11-21T19:39:17.958ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
58422024-11-21T19:39:17.958ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
58432024-11-21T19:39:17.962ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
58442024-11-21T19:39:17.962ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 129.585481ms iteration = 37 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
58452024-11-21T19:39:17.979ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
58462024-11-21T19:39:17.980ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
58472024-11-21T19:39:17.982ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
58482024-11-21T19:39:17.983ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
58492024-11-21T19:39:17.989ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
58502024-11-21T19:39:17.989ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
58512024-11-21T19:39:17.989ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
58522024-11-21T19:39:17.989ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
58532024-11-21T19:39:17.989ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
58542024-11-21T19:39:17.989ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
58552024-11-21T19:39:17.990ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
58562024-11-21T19:39:17.990ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
58572024-11-21T19:39:17.990ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
58582024-11-21T19:39:17.991ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
58592024-11-21T19:39:17.991ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
58602024-11-21T19:39:17.993ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
58612024-11-21T19:39:17.996ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
58622024-11-21T19:39:17.996ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
58632024-11-21T19:39:17.996ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
58642024-11-21T19:39:17.999ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 165.840676ms iteration = 13 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
58652024-11-21T19:39:18.000ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
58662024-11-21T19:39:18.000ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 167.152873ms iteration = 13 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
58672024-11-21T19:39:18.001ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 167.903313ms iteration = 37 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
58682024-11-21T19:39:18.006ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
58692024-11-21T19:39:18.006ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 0 version = 2
58702024-11-21T19:39:18.006ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
58712024-11-21T19:39:18.006ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
58722024-11-21T19:39:18.009ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
58732024-11-21T19:39:18.014ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
58742024-11-21T19:39:18.014ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
58752024-11-21T19:39:18.014ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
58762024-11-21T19:39:18.017ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 1 version = 2
58772024-11-21T19:39:18.017ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
58782024-11-21T19:39:18.017ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-11-21 19:33:32.758954 UTC dns_group = external generation = 2
58792024-11-21T19:39:18.017ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 184.696697ms iteration = 7 status = {"generation":2}
58802024-11-21T19:39:18.018ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
58812024-11-21T19:39:18.018ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
58822024-11-21T19:39:18.018ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
58832024-11-21T19:39:18.019ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
58842024-11-21T19:39:18.021ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
58852024-11-21T19:39:18.024ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
58862024-11-21T19:39:18.024ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
58872024-11-21T19:39:18.024ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
58882024-11-21T19:39:18.027ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 193.566157ms iteration = 13 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
58892024-11-21T19:39:18.027ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 51 version = 1
58902024-11-21T19:39:18.027ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
58912024-11-21T19:39:18.027ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-11-21 19:33:31.797390 UTC dns_group = internal generation = 1
58922024-11-21T19:39:18.027ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 194.684051ms iteration = 7 status = {"generation":1}
58932024-11-21T19:39:18.030ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
58942024-11-21T19:39:18.030ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
58952024-11-21T19:39:18.031ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
58962024-11-21T19:39:18.031ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
58972024-11-21T19:39:18.031ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 197.922835ms iteration = 13 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
58982024-11-21T19:39:18.035ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
58992024-11-21T19:39:18.037ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
59002024-11-21T19:39:18.037ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
59012024-11-21T19:39:18.037ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
59022024-11-21T19:39:18.038ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
59032024-11-21T19:39:18.042ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
59042024-11-21T19:39:18.042ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
59052024-11-21T19:39:18.043ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
59062024-11-21T19:39:18.045ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-11-21T19:33:15.446964Z, time_modified: 2024-11-21T19:33:15.446964Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }) }
59072024-11-21T19:39:18.045ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 212.311086ms iteration = 7 status = {"by_dns_name":{"default-silo.sys.oxide.test":{"silo_id":"001de000-5110-4000-8000-000000000000","tls_certs":[]},"recovery.sys.oxide.test":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]}},"default_endpoint":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]},"warnings":["silo 001de000-5110-4000-8000-000000000000 with DNS name \\"default-silo.sys.oxide.test\\" has no usable certificates"]}
59082024-11-21T19:39:18.047ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
59092024-11-21T19:39:18.047ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
59102024-11-21T19:39:18.047ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
59112024-11-21T19:39:18.048ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "d976934b-2ae1-4333-b029-29d876fc16f8", "content-length": "26", "date": "Thu, 21 Nov 2024 19:39:17 GMT"} })
59122024-11-21T19:39:18.048ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
59132024-11-21T19:39:18.049ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 216.746837ms iteration = 13 status = 0
59142024-11-21T19:39:18.050ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
59152024-11-21T19:39:18.051ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
59162024-11-21T19:39:18.051ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
59172024-11-21T19:39:18.051ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
59182024-11-21T19:39:18.089ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
59192024-11-21T19:39:18.096ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 263.214587ms iteration = 14 status = {"physical_disks_added":0}
59202024-11-21T19:39:18.096ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
59212024-11-21T19:39:18.097ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
59222024-11-21T19:39:18.097ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "d4486205-f9bb-4534-9593-88f94b422651", "content-length": "26", "date": "Thu, 21 Nov 2024 19:39:17 GMT"} })
59232024-11-21T19:39:18.097ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
59242024-11-21T19:39:18.100ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
59252024-11-21T19:39:18.118ZWARNnexus (BackgroundTasks): instance reincarnation completed with errors background_task = instance_reincarnation file = nexus/src/app/background/tasks/instance_reincarnation.rs:103 instances_changed_state = 0 instances_found = 0 instances_reincarnated = 0 query_errors = 0 restart_errors = 0
59262024-11-21T19:39:18.118ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_reincarnation elapsed = 284.928877ms iteration = 7 status = {"changed_state":[],"disabled":false,"errors":[],"instances_found":{"Failed":0,"SagaUnwound":0},"instances_reincarnated":[],"restart_errors":[]}
59272024-11-21T19:39:18.144ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
59282024-11-21T19:39:18.144ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
59292024-11-21T19:39:18.145ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "8ebed13e-5e94-47c7-9447-371a65ea05c1", "content-length": "26", "date": "Thu, 21 Nov 2024 19:39:17 GMT"} })
59302024-11-21T19:39:18.145ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
59312024-11-21T19:39:18.146ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
59322024-11-21T19:39:18.195ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
59332024-11-21T19:39:18.195ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::8]:5353/config", status: 204, headers: {"x-request-id": "094ad9bd-81c8-48bc-a2bd-d04506f91680", "date": "Thu, 21 Nov 2024 19:39:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
59342024-11-21T19:39:18.195ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
59352024-11-21T19:39:18.197ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::9]:5353/config", status: 204, headers: {"x-request-id": "ea41044f-8837-47eb-a5e4-467931217e93", "date": "Thu, 21 Nov 2024 19:39:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
59362024-11-21T19:39:18.197ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
59372024-11-21T19:39:18.199ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
59382024-11-21T19:39:18.199ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 366.72314ms iteration = 9 status = {"generation":2,"server_results":{"[fd00:1122:3344:101::8]:5353":{"Ok":null},"[fd00:1122:3344:101::9]:5353":{"Ok":null}}}
59392024-11-21T19:39:18.224ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
59402024-11-21T19:39:18.224ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "28f05356-3153-42b6-81b4-a6ed057def90", "content-length": "1", "date": "Thu, 21 Nov 2024 19:39:18 GMT"} })
59412024-11-21T19:39:18.226ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
59422024-11-21T19:39:18.226ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
59432024-11-21T19:39:18.229ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 397.058706ms iteration = 13 status = null
59442024-11-21T19:39:18.245ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
59452024-11-21T19:39:18.245ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
59462024-11-21T19:39:18.245ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "77d32788-7ecd-47f6-a69e-0994fd36b89f", "content-length": "372", "date": "Thu, 21 Nov 2024 19:39:18 GMT"} })
59472024-11-21T19:39:18.245ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
59482024-11-21T19:39:18.245ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
59492024-11-21T19:39:18.259ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
59502024-11-21T19:39:18.259ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
59512024-11-21T19:39:18.259ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "75e3a68e-3411-43c2-8d45-6414ddd0da81", "date": "Thu, 21 Nov 2024 19:39:18 GMT"} })
59522024-11-21T19:39:18.259ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
59532024-11-21T19:39:18.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
59542024-11-21T19:39:18.260ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 427.559443ms iteration = 13 status = {}
59552024-11-21T19:39:18.262ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
59562024-11-21T19:39:18.262ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
59572024-11-21T19:39:18.265ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
59582024-11-21T19:39:18.265ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
59592024-11-21T19:39:18.267ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
59602024-11-21T19:39:18.287ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
59612024-11-21T19:39:18.290ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "1d3a6082-3ae1-41f3-9b90-cd64cd35cb14", "content-length": "691", "date": "Thu, 21 Nov 2024 19:39:18 GMT"} })
59622024-11-21T19:39:18.290ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
59632024-11-21T19:39:18.290ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
59642024-11-21T19:39:18.290ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
59652024-11-21T19:39:18.290ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
59662024-11-21T19:39:18.290ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
59672024-11-21T19:39:18.291ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "8c545a6c-f4fd-4319-b21b-3e70176de975", "date": "Thu, 21 Nov 2024 19:39:18 GMT"} })
59682024-11-21T19:39:18.293ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 460.175519ms iteration = 13 status = {}
59692024-11-21T19:39:18.293ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 459.434649ms iteration = 13 status = {}
59702024-11-21T19:39:18.332ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
59712024-11-21T19:39:18.332ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:1::1]:5353/config", status: 204, headers: {"x-request-id": "c2ba975e-d284-43d7-8255-0e5d6999c125", "date": "Thu, 21 Nov 2024 19:39:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
59722024-11-21T19:39:18.332ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
59732024-11-21T19:39:18.334ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:3::1]:5353/config", status: 204, headers: {"x-request-id": "5b5fcbc3-f21b-4717-ba2f-7f9117388466", "date": "Thu, 21 Nov 2024 19:39:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
59742024-11-21T19:39:18.334ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
59752024-11-21T19:39:18.336ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:2::1]:5353/config", status: 204, headers: {"x-request-id": "d8ab6dea-fdd9-447e-870e-6a6ab6a6a55f", "date": "Thu, 21 Nov 2024 19:39:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
59762024-11-21T19:39:18.336ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
59772024-11-21T19:39:18.337ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
59782024-11-21T19:39:18.338ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 505.072139ms iteration = 9 status = {"generation":1,"server_results":{"[fd00:1122:3344:1::1]:5353":{"Ok":null},"[fd00:1122:3344:2::1]:5353":{"Ok":null},"[fd00:1122:3344:3::1]:5353":{"Ok":null}}}
59792024-11-21T19:39:18.371ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
59802024-11-21T19:39:18.371ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
59812024-11-21T19:39:18.373ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
59822024-11-21T19:39:18.373ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
59832024-11-21T19:39:18.374ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
59842024-11-21T19:39:18.381ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
59852024-11-21T19:39:18.381ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "c112d4b2-83ed-4db0-96ef-f9153fd94405", "content-length": "134", "date": "Thu, 21 Nov 2024 19:39:18 GMT"} })
59862024-11-21T19:39:18.381ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
59872024-11-21T19:39:18.381ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
59882024-11-21T19:39:18.381ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
59892024-11-21T19:39:18.382ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
59902024-11-21T19:39:18.382ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
59912024-11-21T19:39:18.382ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
59922024-11-21T19:39:18.382ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
59932024-11-21T19:39:18.382ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
59942024-11-21T19:39:18.382ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
59952024-11-21T19:39:18.382ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
59962024-11-21T19:39:18.382ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
59972024-11-21T19:39:18.382ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
59982024-11-21T19:39:18.382ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
59992024-11-21T19:39:18.383ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
60002024-11-21T19:39:18.384ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "be9966b4-bab0-40eb-a8e5-7edc699dda90", "content-length": "2", "date": "Thu, 21 Nov 2024 19:39:18 GMT"} })
60012024-11-21T19:39:18.384ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
60022024-11-21T19:39:18.384ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "54480fc4-01c1-4b84-aa38-3ec8716bfdfe", "content-length": "51", "date": "Thu, 21 Nov 2024 19:39:18 GMT"} })
60032024-11-21T19:39:18.384ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
60042024-11-21T19:39:18.384ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
60052024-11-21T19:39:18.436ZDEBGnexus (producer-registration-task): registering / renewing oximeter producer lease with Nexus
60062024-11-21T19:39:18.436ZDEBGnexus (producer-registration-task): using nexus address for registration addr = [fd00:1122:3344:101::c]:12221
60072024-11-21T19:39:18.467ZDEBGnexus (producer-registration-task): client request body = Some(Body) method = POST uri = http://[fd00:1122:3344:101::c]:12221/metrics/producers
60082024-11-21T19:39:18.467ZINFOnexus (dropshot_internal): accepted connection file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023 local_addr = [fd00:1122:3344:101::c]:12221 remote_addr = [fd00:1122:3344:101::c]:33263
60092024-11-21T19:39:18.467ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:33263 req_id = 4f68b665-8d3e-42bb-ac97-a253f6149e0b roles = RoleSet { roles: {} } uri = /metrics/producers
60102024-11-21T19:39:18.467ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:33263 req_id = 4f68b665-8d3e-42bb-ac97-a253f6149e0b resource = Database result = Ok(()) uri = /metrics/producers
60112024-11-21T19:39:18.475ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "2fb87d6a-7697-4761-ab9a-c19f29ac0d0e", "date": "Thu, 21 Nov 2024 19:39:18 GMT"} })
60122024-11-21T19:39:18.475ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
60132024-11-21T19:39:18.475ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
60142024-11-21T19:39:18.476ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "9f1a1dec-d8af-48e1-aa51-b919c6fc5f2c", "content-length": "682", "date": "Thu, 21 Nov 2024 19:39:18 GMT"} })
60152024-11-21T19:39:18.476ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
60162024-11-21T19:39:18.476ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
60172024-11-21T19:39:18.479ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
60182024-11-21T19:39:18.479ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
60192024-11-21T19:39:18.483ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
60202024-11-21T19:39:18.483ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
60212024-11-21T19:39:18.485ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
60222024-11-21T19:39:18.485ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
60232024-11-21T19:39:18.492ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 659.31922ms iteration = 13 status = {}
60242024-11-21T19:39:18.506ZINFOnexus: assigned collector to new producer collector_id = 5fb99117-c685-44d5-bec2-c58adb63b619 file = nexus/src/app/oximeter.rs:76 producer_id = 915e11b8-d7cb-4730-9906-7980540813dc
60252024-11-21T19:39:18.506ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 38423 local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:33263 req_id = 4f68b665-8d3e-42bb-ac97-a253f6149e0b response_code = 201 uri = /metrics/producers
60262024-11-21T19:39:18.506ZDEBGnexus (producer-registration-task): client response result = Ok(Response { url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", status: 201, headers: {"content-type": "application/json", "x-request-id": "4f68b665-8d3e-42bb-ac97-a253f6149e0b", "content-length": "41", "date": "Thu, 21 Nov 2024 19:39:18 GMT"} })
60272024-11-21T19:39:18.507ZDEBGnexus (producer-registration-task): registered with nexus successfully
60282024-11-21T19:39:18.507ZDEBGnexus (producer-registration-task): pausing until time to renew lease lease_duration = 600s wait_period = 150s
60292024-11-21T19:39:26.961ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 373 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 0ad3f4e2-eb26-4edd-aee7-ce01e14caee3 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
60302024-11-21T19:39:27.833ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 38 reason = Timeout
60312024-11-21T19:39:27.833ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
60322024-11-21T19:39:27.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
60332024-11-21T19:39:27.834ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 38 reason = Timeout
60342024-11-21T19:39:27.834ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
60352024-11-21T19:39:27.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
60362024-11-21T19:39:27.835ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
60372024-11-21T19:39:27.836ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
60382024-11-21T19:39:27.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
60392024-11-21T19:39:27.838ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
60402024-11-21T19:39:27.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
60412024-11-21T19:39:27.838ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 4.608462ms iteration = 38 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
60422024-11-21T19:39:27.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
60432024-11-21T19:39:27.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
60442024-11-21T19:39:27.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
60452024-11-21T19:39:27.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
60462024-11-21T19:39:27.844ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
60472024-11-21T19:39:27.845ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
60482024-11-21T19:39:27.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
60492024-11-21T19:39:27.856ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
60502024-11-21T19:39:27.856ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 22.39221ms iteration = 38 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
60512024-11-21T19:39:31.953ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = bcdd3404-d48f-4814-8436-12a9a809e9d6 roles = RoleSet { roles: {} } uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
60522024-11-21T19:39:31.954ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = bcdd3404-d48f-4814-8436-12a9a809e9d6 resource = Database result = Ok(()) uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
60532024-11-21T19:39:31.957ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 3116 local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = bcdd3404-d48f-4814-8436-12a9a809e9d6 response_code = 200 uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
60542024-11-21T19:39:31.957ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = b6abfde0-3add-4835-be35-7426126ae4d1 roles = RoleSet { roles: {} } uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
60552024-11-21T19:39:31.957ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = b6abfde0-3add-4835-be35-7426126ae4d1 resource = Database result = Ok(()) uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
60562024-11-21T19:39:31.959ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 2290 local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = b6abfde0-3add-4835-be35-7426126ae4d1 response_code = 200 uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
60572024-11-21T19:39:37.833ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 39 reason = Timeout
60582024-11-21T19:39:37.833ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
60592024-11-21T19:39:37.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
60602024-11-21T19:39:37.834ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 39 reason = Timeout
60612024-11-21T19:39:37.834ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
60622024-11-21T19:39:37.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
60632024-11-21T19:39:37.835ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
60642024-11-21T19:39:37.836ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
60652024-11-21T19:39:37.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
60662024-11-21T19:39:37.838ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 3.968304ms iteration = 39 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
60672024-11-21T19:39:37.838ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
60682024-11-21T19:39:37.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
60692024-11-21T19:39:37.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
60702024-11-21T19:39:37.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
60712024-11-21T19:39:37.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
60722024-11-21T19:39:37.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
60732024-11-21T19:39:37.845ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
60742024-11-21T19:39:37.845ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
60752024-11-21T19:39:37.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
60762024-11-21T19:39:37.855ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
60772024-11-21T19:39:37.855ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 22.281737ms iteration = 39 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
60782024-11-21T19:39:41.961ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 303 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = bbb6e2e7-4da6-4504-a4ee-5e35f4cafc1a response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
60792024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 14 reason = Timeout
60802024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
60812024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 14 reason = Timeout
60822024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 14 reason = Timeout
60832024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 15 reason = Timeout
60842024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 14 reason = Timeout
60852024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 40 reason = Timeout
60862024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
60872024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
60882024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
60892024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 14 reason = Timeout
60902024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
60912024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
60922024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 14 reason = Timeout
60932024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
60942024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 14 reason = Timeout
60952024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
60962024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
60972024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
60982024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
60992024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
61002024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
61012024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
61022024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
61032024-11-21T19:39:47.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
61042024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 14 reason = Timeout
61052024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 14 reason = Timeout
61062024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
61072024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
61082024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 40 reason = Timeout
61092024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
61102024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 14 reason = Timeout
61112024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
61122024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 14 reason = Timeout
61132024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
61142024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
61152024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 14 reason = Timeout
61162024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
61172024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 14 reason = Timeout
61182024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
61192024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 14 reason = Timeout
61202024-11-21T19:39:47.834ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
61212024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
61222024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
61232024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
61242024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
61252024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
61262024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
61272024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
61282024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
61292024-11-21T19:39:47.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
61302024-11-21T19:39:47.835ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1130, tv_nsec: 711155915 } })
61312024-11-21T19:39:47.836ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
61322024-11-21T19:39:47.836ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
61332024-11-21T19:39:47.836ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.453096ms iteration = 14 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
61342024-11-21T19:39:47.836ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1130, tv_nsec: 712079767 } })
61352024-11-21T19:39:47.837ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
61362024-11-21T19:39:47.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
61372024-11-21T19:39:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
61382024-11-21T19:39:47.840ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
61392024-11-21T19:39:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
61402024-11-21T19:39:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
61412024-11-21T19:39:47.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
61422024-11-21T19:39:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
61432024-11-21T19:39:47.841ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
61442024-11-21T19:39:47.842ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
61452024-11-21T19:39:47.844ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
61462024-11-21T19:39:47.844ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
61472024-11-21T19:39:47.844ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
61482024-11-21T19:39:47.844ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
61492024-11-21T19:39:47.844ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
61502024-11-21T19:39:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
61512024-11-21T19:39:47.849ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
61522024-11-21T19:39:47.849ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
61532024-11-21T19:39:47.850ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 15.71973ms iteration = 14 status = {"errors":[],"records_set_to_done":[]}
61542024-11-21T19:39:47.850ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
61552024-11-21T19:39:47.850ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
61562024-11-21T19:39:47.850ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
61572024-11-21T19:39:47.851ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
61582024-11-21T19:39:47.852ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
61592024-11-21T19:39:47.852ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
61602024-11-21T19:39:47.852ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
61612024-11-21T19:39:47.852ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
61622024-11-21T19:39:47.852ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
61632024-11-21T19:39:47.853ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
61642024-11-21T19:39:47.853ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
61652024-11-21T19:39:47.853ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
61662024-11-21T19:39:47.855ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
61672024-11-21T19:39:47.855ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
61682024-11-21T19:39:47.856ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
61692024-11-21T19:39:47.856ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
61702024-11-21T19:39:47.856ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1130, tv_nsec: 732095395 } })
61712024-11-21T19:39:47.857ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
61722024-11-21T19:39:47.861ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
61732024-11-21T19:39:47.861ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
61742024-11-21T19:39:47.861ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
61752024-11-21T19:39:47.865ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
61762024-11-21T19:39:47.865ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
61772024-11-21T19:39:47.865ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
61782024-11-21T19:39:47.865ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
61792024-11-21T19:39:47.865ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
61802024-11-21T19:39:47.865ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
61812024-11-21T19:39:47.865ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
61822024-11-21T19:39:47.865ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
61832024-11-21T19:39:47.866ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
61842024-11-21T19:39:47.867ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
61852024-11-21T19:39:47.868ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
61862024-11-21T19:39:47.869ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
61872024-11-21T19:39:47.871ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
61882024-11-21T19:39:47.871ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
61892024-11-21T19:39:47.871ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
61902024-11-21T19:39:47.873ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
61912024-11-21T19:39:47.875ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
61922024-11-21T19:39:47.875ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
61932024-11-21T19:39:47.875ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
61942024-11-21T19:39:47.880ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
61952024-11-21T19:39:47.880ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
61962024-11-21T19:39:47.880ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
61972024-11-21T19:39:47.880ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
61982024-11-21T19:39:47.881ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
61992024-11-21T19:39:47.881ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 48.6555ms iteration = 40 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
62002024-11-21T19:39:47.884ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
62012024-11-21T19:39:47.884ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
62022024-11-21T19:39:47.887ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 52.991168ms iteration = 14 status = {"errors":[],"garbage_collect_requested":[]}
62032024-11-21T19:39:47.889ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
62042024-11-21T19:39:47.889ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
62052024-11-21T19:39:47.892ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
62062024-11-21T19:39:47.892ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
62072024-11-21T19:39:47.892ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
62082024-11-21T19:39:47.892ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
62092024-11-21T19:39:47.893ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 58.839736ms iteration = 14 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
62102024-11-21T19:39:47.899ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
62112024-11-21T19:39:47.899ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 65.16554ms iteration = 14 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
62122024-11-21T19:39:47.937ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
62132024-11-21T19:39:47.937ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
62142024-11-21T19:39:47.938ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "4240cfcd-081b-446a-85eb-2d8b8039ffbc", "content-length": "26", "date": "Thu, 21 Nov 2024 19:39:47 GMT"} })
62152024-11-21T19:39:47.938ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
62162024-11-21T19:39:47.939ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
62172024-11-21T19:39:47.939ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
62182024-11-21T19:39:47.939ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "f9616747-8861-4acb-ad66-7c2ae60f1b55", "content-length": "26", "date": "Thu, 21 Nov 2024 19:39:47 GMT"} })
62192024-11-21T19:39:47.939ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
62202024-11-21T19:39:47.940ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
62212024-11-21T19:39:47.941ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
62222024-11-21T19:39:47.960ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 127.567514ms iteration = 15 status = {"physical_disks_added":0}
62232024-11-21T19:39:47.992ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
62242024-11-21T19:39:47.992ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
62252024-11-21T19:39:47.994ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
62262024-11-21T19:39:47.994ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
62272024-11-21T19:39:47.999ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "bf30f967-8110-4f4a-ae72-f187800dd1ed", "content-length": "372", "date": "Thu, 21 Nov 2024 19:39:47 GMT"} })
62282024-11-21T19:39:47.999ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
62292024-11-21T19:39:47.999ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
62302024-11-21T19:39:48.017ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
62312024-11-21T19:39:48.017ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
62322024-11-21T19:39:48.017ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "48ce56ac-18a0-4adf-8a21-5d5659db8564", "content-length": "26", "date": "Thu, 21 Nov 2024 19:39:47 GMT"} })
62332024-11-21T19:39:48.017ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
62342024-11-21T19:39:48.019ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
62352024-11-21T19:39:48.027ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 194.477588ms iteration = 14 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
62362024-11-21T19:39:48.058ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
62372024-11-21T19:39:48.059ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "35f285c1-a1eb-4360-b4d9-cfc448c69075", "content-length": "1", "date": "Thu, 21 Nov 2024 19:39:47 GMT"} })
62382024-11-21T19:39:48.060ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
62392024-11-21T19:39:48.061ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
62402024-11-21T19:39:48.156ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 323.25525ms iteration = 14 status = {}
62412024-11-21T19:39:48.164ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
62422024-11-21T19:39:48.164ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "387db6cf-df41-43e0-91f3-59cf6d554193", "content-length": "691", "date": "Thu, 21 Nov 2024 19:39:47 GMT"} })
62432024-11-21T19:39:48.164ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
62442024-11-21T19:39:48.164ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
62452024-11-21T19:39:48.164ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
62462024-11-21T19:39:48.166ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 333.251663ms iteration = 14 status = {}
62472024-11-21T19:39:48.166ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
62482024-11-21T19:39:48.166ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 332.011476ms iteration = 40 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
62492024-11-21T19:39:48.166ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
62502024-11-21T19:39:48.167ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 332.750536ms iteration = 14 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
62512024-11-21T19:39:48.168ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
62522024-11-21T19:39:48.168ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 335.815427ms iteration = 14 status = null
62532024-11-21T19:39:48.171ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
62542024-11-21T19:39:48.171ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
62552024-11-21T19:39:48.171ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
62562024-11-21T19:39:48.173ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 340.044924ms iteration = 14 status = 0
62572024-11-21T19:39:48.201ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
62582024-11-21T19:39:48.201ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
62592024-11-21T19:39:48.201ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
62602024-11-21T19:39:48.201ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
62612024-11-21T19:39:48.208ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
62622024-11-21T19:39:48.208ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
62632024-11-21T19:39:48.210ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
62642024-11-21T19:39:48.210ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
62652024-11-21T19:39:48.210ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
62662024-11-21T19:39:48.212ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
62672024-11-21T19:39:48.212ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
62682024-11-21T19:39:48.212ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "d1326a7e-730d-4ff1-a865-1a05ab706dd3", "date": "Thu, 21 Nov 2024 19:39:47 GMT"} })
62692024-11-21T19:39:48.212ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
62702024-11-21T19:39:48.212ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
62712024-11-21T19:39:48.215ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
62722024-11-21T19:39:48.215ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
62732024-11-21T19:39:48.218ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
62742024-11-21T19:39:48.218ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
62752024-11-21T19:39:48.220ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
62762024-11-21T19:39:48.221ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
62772024-11-21T19:39:48.221ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
62782024-11-21T19:39:48.221ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "addade80-8c6b-4eb1-b81e-284467ec608c", "date": "Thu, 21 Nov 2024 19:39:47 GMT"} })
62792024-11-21T19:39:48.223ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 388.846185ms iteration = 14 status = {}
62802024-11-21T19:39:48.223ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "8998f093-581b-47ef-aa26-177e84af76ee", "content-length": "134", "date": "Thu, 21 Nov 2024 19:39:47 GMT"} })
62812024-11-21T19:39:48.223ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
62822024-11-21T19:39:48.223ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
62832024-11-21T19:39:48.223ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
62842024-11-21T19:39:48.223ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
62852024-11-21T19:39:48.223ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
62862024-11-21T19:39:48.223ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
62872024-11-21T19:39:48.223ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
62882024-11-21T19:39:48.223ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
62892024-11-21T19:39:48.224ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
62902024-11-21T19:39:48.224ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
62912024-11-21T19:39:48.224ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
62922024-11-21T19:39:48.224ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
62932024-11-21T19:39:48.224ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
62942024-11-21T19:39:48.226ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
62952024-11-21T19:39:48.226ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "9105f516-abed-4657-ae34-ce078d8db8e7", "content-length": "2", "date": "Thu, 21 Nov 2024 19:39:47 GMT"} })
62962024-11-21T19:39:48.226ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
62972024-11-21T19:39:48.226ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "1a87511e-4000-4849-8c73-ebcdcfed51f4", "content-length": "51", "date": "Thu, 21 Nov 2024 19:39:47 GMT"} })
62982024-11-21T19:39:48.226ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
62992024-11-21T19:39:48.226ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
63002024-11-21T19:39:48.235ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
63012024-11-21T19:39:48.235ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
63022024-11-21T19:39:48.235ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 400.859026ms iteration = 14 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
63032024-11-21T19:39:48.327ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "f64f3072-c6a0-4670-a567-318702edba16", "date": "Thu, 21 Nov 2024 19:39:47 GMT"} })
63042024-11-21T19:39:48.327ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
63052024-11-21T19:39:48.327ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
63062024-11-21T19:39:48.328ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "211ab2d4-be28-4054-990e-15a86e338595", "content-length": "682", "date": "Thu, 21 Nov 2024 19:39:47 GMT"} })
63072024-11-21T19:39:48.328ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
63082024-11-21T19:39:48.328ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
63092024-11-21T19:39:48.331ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
63102024-11-21T19:39:48.331ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
63112024-11-21T19:39:48.335ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
63122024-11-21T19:39:48.335ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
63132024-11-21T19:39:48.337ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
63142024-11-21T19:39:48.338ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
63152024-11-21T19:39:48.344ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 511.241531ms iteration = 14 status = {}
63162024-11-21T19:39:56.963ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 310 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = ece0b27d-19a5-4c3a-84f3-f04304eb0852 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
63172024-11-21T19:39:57.833ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 41 reason = Timeout
63182024-11-21T19:39:57.833ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
63192024-11-21T19:39:57.833ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
63202024-11-21T19:39:57.834ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 41 reason = Timeout
63212024-11-21T19:39:57.834ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
63222024-11-21T19:39:57.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
63232024-11-21T19:39:57.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
63242024-11-21T19:39:57.836ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
63252024-11-21T19:39:57.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
63262024-11-21T19:39:57.837ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
63272024-11-21T19:39:57.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
63282024-11-21T19:39:57.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
63292024-11-21T19:39:57.839ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
63302024-11-21T19:39:57.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
63312024-11-21T19:39:57.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
63322024-11-21T19:39:57.843ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
63332024-11-21T19:39:57.843ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
63342024-11-21T19:39:57.843ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
63352024-11-21T19:39:57.854ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
63362024-11-21T19:39:57.854ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 21.486657ms iteration = 41 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
63372024-11-21T19:39:57.870ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 36.085312ms iteration = 41 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
63382024-11-21T19:40:07.834ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 42 reason = Timeout
63392024-11-21T19:40:07.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
63402024-11-21T19:40:07.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
63412024-11-21T19:40:07.834ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 42 reason = Timeout
63422024-11-21T19:40:07.834ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
63432024-11-21T19:40:07.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
63442024-11-21T19:40:07.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
63452024-11-21T19:40:07.836ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
63462024-11-21T19:40:07.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
63472024-11-21T19:40:07.839ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 4.4657ms iteration = 42 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
63482024-11-21T19:40:07.839ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
63492024-11-21T19:40:07.839ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
63502024-11-21T19:40:07.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
63512024-11-21T19:40:07.841ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
63522024-11-21T19:40:07.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
63532024-11-21T19:40:07.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
63542024-11-21T19:40:07.845ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
63552024-11-21T19:40:07.845ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
63562024-11-21T19:40:07.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
63572024-11-21T19:40:07.859ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
63582024-11-21T19:40:07.859ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 25.549912ms iteration = 42 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
63592024-11-21T19:40:11.961ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 282 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 327bdd8a-be09-47ab-90e8-e22f5741cdd9 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
63602024-11-21T19:40:16.955ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = 66c70805-1dbf-4fd4-a43d-8a9ce7cb0f20 roles = RoleSet { roles: {} } uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
63612024-11-21T19:40:16.955ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = 66c70805-1dbf-4fd4-a43d-8a9ce7cb0f20 resource = Database result = Ok(()) uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
63622024-11-21T19:40:16.957ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 2855 local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = 66c70805-1dbf-4fd4-a43d-8a9ce7cb0f20 response_code = 200 uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
63632024-11-21T19:40:16.958ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = 59bf8ae2-828a-4d7a-9a89-43601a42404c roles = RoleSet { roles: {} } uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
63642024-11-21T19:40:16.958ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = 59bf8ae2-828a-4d7a-9a89-43601a42404c resource = Database result = Ok(()) uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
63652024-11-21T19:40:16.960ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 2582 local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = 59bf8ae2-828a-4d7a-9a89-43601a42404c response_code = 200 uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
63662024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 15 reason = Timeout
63672024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 43 reason = Timeout
63682024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
63692024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
63702024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 8 reason = Timeout
63712024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
63722024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 8 reason = Timeout
63732024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
63742024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 16 reason = Timeout
63752024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
63762024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 10 reason = Timeout
63772024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 10 reason = Timeout
63782024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 15 reason = Timeout
63792024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 15 reason = Timeout
63802024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 8 reason = Timeout
63812024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
63822024-11-21T19:40:17.833ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-11-21 19:30:17.833833376 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
63832024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
63842024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 9 reason = Timeout
63852024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 15 reason = Timeout
63862024-11-21T19:40:17.833ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:83 reason = blueprint disabled target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb
63872024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 15 reason = Timeout
63882024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 15.84\u{b5}s iteration = 9 status = {"enabled":false,"target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb"}
63892024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
63902024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
63912024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 8 reason = Timeout
63922024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 8 reason = Timeout
63932024-11-21T19:40:17.833ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
63942024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
63952024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
63962024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
63972024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 15 reason = Timeout
63982024-11-21T19:40:17.833ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 8 reason = Timeout
63992024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
64002024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 8 reason = Timeout
64012024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
64022024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 15 reason = Timeout
64032024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
64042024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
64052024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
64062024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
64072024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
64082024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
64092024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
64102024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
64112024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
64122024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
64132024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
64142024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
64152024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
64162024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
64172024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 8 reason = Timeout
64182024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 43 reason = Timeout
64192024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 15 reason = Timeout
64202024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 15 reason = Timeout
64212024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
64222024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): activating background_task = instance_reincarnation iteration = 8 reason = Timeout
64232024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
64242024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
64252024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
64262024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
64272024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 8 reason = Timeout
64282024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
64292024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 15 reason = Timeout
64302024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 15 reason = Timeout
64312024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
64322024-11-21T19:40:17.834ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
64332024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
64342024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 15 reason = Timeout
64352024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
64362024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 15 reason = Timeout
64372024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
64382024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 15 reason = Timeout
64392024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
64402024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
64412024-11-21T19:40:17.834ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("4e669ff4-6dde-4cd2-8491-623460b56d8d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 1160, tv_nsec: 709506943 } })
64422024-11-21T19:40:17.834ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0377cf96-baf3-4e09-a570-eb7c67037d4d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0d1b3b31-4591-4bcb-931d-e8e03365f0d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:1::1))) }], valid_until: Instant { tv_sec: 1160, tv_nsec: 709524834 } })
64432024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
64442024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
64452024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
64462024-11-21T19:40:17.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
64472024-11-21T19:40:17.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
64482024-11-21T19:40:17.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
64492024-11-21T19:40:17.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
64502024-11-21T19:40:17.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
64512024-11-21T19:40:17.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
64522024-11-21T19:40:17.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
64532024-11-21T19:40:17.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
64542024-11-21T19:40:17.835ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
64552024-11-21T19:40:17.835ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 1.435679ms iteration = 8 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
64562024-11-21T19:40:17.835ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } dns_group = internal
64572024-11-21T19:40:17.835ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 1.715833ms iteration = 8 status = {"addresses":["[fd00:1122:3344:3::1]:5353","[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353"]}
64582024-11-21T19:40:17.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
64592024-11-21T19:40:17.837ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
64602024-11-21T19:40:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
64612024-11-21T19:40:17.837ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
64622024-11-21T19:40:17.838ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1160, tv_nsec: 713021610 } })
64632024-11-21T19:40:17.838ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
64642024-11-21T19:40:17.838ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 4.884555ms iteration = 15 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
64652024-11-21T19:40:17.838ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
64662024-11-21T19:40:17.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
64672024-11-21T19:40:17.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
64682024-11-21T19:40:17.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
64692024-11-21T19:40:17.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
64702024-11-21T19:40:17.840ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
64712024-11-21T19:40:17.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
64722024-11-21T19:40:17.841ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
64732024-11-21T19:40:17.841ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
64742024-11-21T19:40:17.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
64752024-11-21T19:40:17.841ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
64762024-11-21T19:40:17.842ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
64772024-11-21T19:40:17.843ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
64782024-11-21T19:40:17.843ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
64792024-11-21T19:40:17.843ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1160, tv_nsec: 718174469 } })
64802024-11-21T19:40:17.843ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
64812024-11-21T19:40:17.843ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
64822024-11-21T19:40:17.843ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
64832024-11-21T19:40:17.844ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
64842024-11-21T19:40:17.844ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-11-21 19:30:17.833833376 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
64852024-11-21T19:40:17.844ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 10.214647ms iteration = 8 status = {"expiration":"2024-11-21T19:30:17.833833376Z","pruned":[]}
64862024-11-21T19:40:17.845ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
64872024-11-21T19:40:17.845ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
64882024-11-21T19:40:17.845ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
64892024-11-21T19:40:17.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
64902024-11-21T19:40:17.847ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
64912024-11-21T19:40:17.847ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
64922024-11-21T19:40:17.847ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
64932024-11-21T19:40:17.862ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0 total = 0
64942024-11-21T19:40:17.862ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:171 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0 vmms_found = 0
64952024-11-21T19:40:17.862ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 27.896862ms iteration = 8 status = {"errors":[],"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0,"vmms_found":0}
64962024-11-21T19:40:17.864ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 29.444863ms iteration = 8 status = {"errors":[],"found_port_ok":[]}
64972024-11-21T19:40:17.870ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1160, tv_nsec: 745171240 } })
64982024-11-21T19:40:17.872ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
64992024-11-21T19:40:17.873ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
65002024-11-21T19:40:17.873ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
65012024-11-21T19:40:17.874ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
65022024-11-21T19:40:17.874ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
65032024-11-21T19:40:17.874ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
65042024-11-21T19:40:17.875ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
65052024-11-21T19:40:17.876ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
65062024-11-21T19:40:17.876ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
65072024-11-21T19:40:17.878ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
65082024-11-21T19:40:17.883ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
65092024-11-21T19:40:17.883ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
65102024-11-21T19:40:17.883ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
65112024-11-21T19:40:17.906ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
65122024-11-21T19:40:17.907ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
65132024-11-21T19:40:17.909ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
65142024-11-21T19:40:17.911ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
65152024-11-21T19:40:17.911ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
65162024-11-21T19:40:17.912ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
65172024-11-21T19:40:17.913ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
65182024-11-21T19:40:17.913ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
65192024-11-21T19:40:17.913ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
65202024-11-21T19:40:17.913ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
65212024-11-21T19:40:17.914ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
65222024-11-21T19:40:17.914ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
65232024-11-21T19:40:17.914ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
65242024-11-21T19:40:17.914ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
65252024-11-21T19:40:17.915ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
65262024-11-21T19:40:17.915ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
65272024-11-21T19:40:17.915ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
65282024-11-21T19:40:17.917ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
65292024-11-21T19:40:17.917ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
65302024-11-21T19:40:17.917ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
65312024-11-21T19:40:17.918ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
65322024-11-21T19:40:17.918ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
65332024-11-21T19:40:17.918ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
65342024-11-21T19:40:17.918ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
65352024-11-21T19:40:17.918ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
65362024-11-21T19:40:17.918ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
65372024-11-21T19:40:17.919ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 84.64964ms iteration = 15 status = {"errors":[],"garbage_collect_requested":[]}
65382024-11-21T19:40:17.920ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
65392024-11-21T19:40:17.920ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
65402024-11-21T19:40:17.920ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
65412024-11-21T19:40:17.921ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 86.527506ms iteration = 15 status = {"errors":[],"records_set_to_done":[]}
65422024-11-21T19:40:17.922ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
65432024-11-21T19:40:17.924ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
65442024-11-21T19:40:17.924ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
65452024-11-21T19:40:17.925ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
65462024-11-21T19:40:17.927ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
65472024-11-21T19:40:17.927ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
65482024-11-21T19:40:17.927ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
65492024-11-21T19:40:17.928ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
65502024-11-21T19:40:17.928ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
65512024-11-21T19:40:17.928ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
65522024-11-21T19:40:17.928ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
65532024-11-21T19:40:17.928ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
65542024-11-21T19:40:17.929ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
65552024-11-21T19:40:17.930ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
65562024-11-21T19:40:17.930ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
65572024-11-21T19:40:17.930ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
65582024-11-21T19:40:17.930ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
65592024-11-21T19:40:17.930ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
65602024-11-21T19:40:17.930ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
65612024-11-21T19:40:17.930ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
65622024-11-21T19:40:17.931ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
65632024-11-21T19:40:17.931ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
65642024-11-21T19:40:17.931ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
65652024-11-21T19:40:17.936ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
65662024-11-21T19:40:17.937ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
65672024-11-21T19:40:17.937ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 103.081538ms iteration = 8 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
65682024-11-21T19:40:17.939ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
65692024-11-21T19:40:17.939ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
65702024-11-21T19:40:17.941ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
65712024-11-21T19:40:17.942ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
65722024-11-21T19:40:17.942ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
65732024-11-21T19:40:17.942ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 108.288707ms iteration = 15 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
65742024-11-21T19:40:17.944ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
65752024-11-21T19:40:17.944ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
65762024-11-21T19:40:17.944ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
65772024-11-21T19:40:17.945ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
65782024-11-21T19:40:17.945ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
65792024-11-21T19:40:17.945ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
65802024-11-21T19:40:17.945ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
65812024-11-21T19:40:17.945ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 112.25704ms iteration = 43 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
65822024-11-21T19:40:17.946ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
65832024-11-21T19:40:17.950ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
65842024-11-21T19:40:17.951ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
65852024-11-21T19:40:17.951ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
65862024-11-21T19:40:17.955ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 121.137428ms iteration = 15 status = 0
65872024-11-21T19:40:17.961ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
65882024-11-21T19:40:17.961ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
65892024-11-21T19:40:17.966ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 132.143155ms iteration = 15 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
65902024-11-21T19:40:17.968ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 134.969544ms iteration = 15 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
65912024-11-21T19:40:17.970ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 136.096098ms iteration = 43 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
65922024-11-21T19:40:17.974ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 141.063015ms iteration = 16 status = {"physical_disks_added":0}
65932024-11-21T19:40:17.981ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
65942024-11-21T19:40:17.981ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
65952024-11-21T19:40:17.981ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
65962024-11-21T19:40:17.981ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
65972024-11-21T19:40:17.983ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 149.322815ms iteration = 15 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
65982024-11-21T19:40:17.992ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
65992024-11-21T19:40:17.992ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
66002024-11-21T19:40:17.995ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
66012024-11-21T19:40:17.997ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
66022024-11-21T19:40:17.998ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
66032024-11-21T19:40:17.998ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
66042024-11-21T19:40:17.998ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
66052024-11-21T19:40:17.998ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
66062024-11-21T19:40:17.998ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
66072024-11-21T19:40:17.998ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
66082024-11-21T19:40:17.999ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
66092024-11-21T19:40:17.999ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 0 version = 2
66102024-11-21T19:40:17.999ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
66112024-11-21T19:40:17.999ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
66122024-11-21T19:40:17.999ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
66132024-11-21T19:40:18.000ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
66142024-11-21T19:40:18.003ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
66152024-11-21T19:40:18.003ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
66162024-11-21T19:40:18.003ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
66172024-11-21T19:40:18.005ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 51 version = 1
66182024-11-21T19:40:18.005ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
66192024-11-21T19:40:18.005ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-11-21 19:33:31.797390 UTC dns_group = internal generation = 1
66202024-11-21T19:40:18.005ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 171.448971ms iteration = 8 status = {"generation":1}
66212024-11-21T19:40:18.007ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
66222024-11-21T19:40:18.012ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
66232024-11-21T19:40:18.012ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
66242024-11-21T19:40:18.012ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "1834d33f-87c7-44d4-8b4a-4ac50166743c", "content-length": "26", "date": "Thu, 21 Nov 2024 19:40:17 GMT"} })
66252024-11-21T19:40:18.012ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
66262024-11-21T19:40:18.014ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
66272024-11-21T19:40:18.029ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
66282024-11-21T19:40:18.029ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
66292024-11-21T19:40:18.029ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 194.58733ms iteration = 15 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
66302024-11-21T19:40:18.032ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
66312024-11-21T19:40:18.035ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
66322024-11-21T19:40:18.035ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
66332024-11-21T19:40:18.035ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
66342024-11-21T19:40:18.037ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 1 version = 2
66352024-11-21T19:40:18.037ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
66362024-11-21T19:40:18.037ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-11-21 19:33:32.758954 UTC dns_group = external generation = 2
66372024-11-21T19:40:18.037ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 204.074056ms iteration = 8 status = {"generation":2}
66382024-11-21T19:40:18.042ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-11-21T19:33:15.446964Z, time_modified: 2024-11-21T19:33:15.446964Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }) }
66392024-11-21T19:40:18.042ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 208.741069ms iteration = 8 status = {"by_dns_name":{"default-silo.sys.oxide.test":{"silo_id":"001de000-5110-4000-8000-000000000000","tls_certs":[]},"recovery.sys.oxide.test":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]}},"default_endpoint":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]},"warnings":["silo 001de000-5110-4000-8000-000000000000 with DNS name \\"default-silo.sys.oxide.test\\" has no usable certificates"]}
66402024-11-21T19:40:18.050ZWARNnexus (BackgroundTasks): instance reincarnation completed with errors background_task = instance_reincarnation file = nexus/src/app/background/tasks/instance_reincarnation.rs:103 instances_changed_state = 0 instances_found = 0 instances_reincarnated = 0 query_errors = 0 restart_errors = 0
66412024-11-21T19:40:18.050ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_reincarnation elapsed = 216.174178ms iteration = 8 status = {"changed_state":[],"disabled":false,"errors":[],"instances_found":{"Failed":0,"SagaUnwound":0},"instances_reincarnated":[],"restart_errors":[]}
66422024-11-21T19:40:18.051ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
66432024-11-21T19:40:18.052ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
66442024-11-21T19:40:18.082ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
66452024-11-21T19:40:18.084ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
66462024-11-21T19:40:18.084ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
66472024-11-21T19:40:18.084ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
66482024-11-21T19:40:18.085ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
66492024-11-21T19:40:18.087ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
66502024-11-21T19:40:18.087ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
66512024-11-21T19:40:18.088ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
66522024-11-21T19:40:18.101ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
66532024-11-21T19:40:18.101ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
66542024-11-21T19:40:18.102ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "98b445ff-489d-426c-9cf8-7972eb0f864c", "content-length": "26", "date": "Thu, 21 Nov 2024 19:40:17 GMT"} })
66552024-11-21T19:40:18.102ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
66562024-11-21T19:40:18.103ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
66572024-11-21T19:40:18.119ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
66582024-11-21T19:40:18.119ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
66592024-11-21T19:40:18.120ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "fb5f9abf-487b-4d64-8a96-14d27bef3f67", "content-length": "26", "date": "Thu, 21 Nov 2024 19:40:17 GMT"} })
66602024-11-21T19:40:18.120ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
66612024-11-21T19:40:18.121ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
66622024-11-21T19:40:18.156ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
66632024-11-21T19:40:18.168ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
66642024-11-21T19:40:18.169ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::8]:5353/config", status: 204, headers: {"x-request-id": "3e5c18fc-7974-4cd3-9160-826db80ed175", "date": "Thu, 21 Nov 2024 19:40:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
66652024-11-21T19:40:18.169ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
66662024-11-21T19:40:18.170ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::9]:5353/config", status: 204, headers: {"x-request-id": "eb914e0e-e4fc-4daf-a5c3-a9b8ed46103a", "date": "Thu, 21 Nov 2024 19:40:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
66672024-11-21T19:40:18.170ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
66682024-11-21T19:40:18.172ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
66692024-11-21T19:40:18.172ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 338.25231ms iteration = 10 status = {"generation":2,"server_results":{"[fd00:1122:3344:101::8]:5353":{"Ok":null},"[fd00:1122:3344:101::9]:5353":{"Ok":null}}}
66702024-11-21T19:40:18.304ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
66712024-11-21T19:40:18.304ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
66722024-11-21T19:40:18.304ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "716fe2cf-e497-4792-8552-7988a4a24b59", "content-length": "372", "date": "Thu, 21 Nov 2024 19:40:17 GMT"} })
66732024-11-21T19:40:18.304ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
66742024-11-21T19:40:18.304ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
66752024-11-21T19:40:18.307ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
66762024-11-21T19:40:18.308ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:1::1]:5353/config", status: 204, headers: {"x-request-id": "dfb060aa-d068-4cd4-8ae6-7b60ba9cdcd2", "date": "Thu, 21 Nov 2024 19:40:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
66772024-11-21T19:40:18.308ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
66782024-11-21T19:40:18.310ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:2::1]:5353/config", status: 204, headers: {"x-request-id": "1ea685a2-76ec-471c-ae18-3a4864176eb8", "date": "Thu, 21 Nov 2024 19:40:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
66792024-11-21T19:40:18.310ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
66802024-11-21T19:40:18.310ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 476.580028ms iteration = 15 status = {}
66812024-11-21T19:40:18.312ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:3::1]:5353/config", status: 204, headers: {"x-request-id": "c49cbcea-c27c-42e6-a7d7-760b5f9bfd17", "date": "Thu, 21 Nov 2024 19:40:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
66822024-11-21T19:40:18.312ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
66832024-11-21T19:40:18.314ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
66842024-11-21T19:40:18.314ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 480.47581ms iteration = 10 status = {"generation":1,"server_results":{"[fd00:1122:3344:1::1]:5353":{"Ok":null},"[fd00:1122:3344:2::1]:5353":{"Ok":null},"[fd00:1122:3344:3::1]:5353":{"Ok":null}}}
66852024-11-21T19:40:18.318ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
66862024-11-21T19:40:18.318ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
66872024-11-21T19:40:18.318ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "99795556-8170-461e-8ad9-e09de51799b4", "date": "Thu, 21 Nov 2024 19:40:17 GMT"} })
66882024-11-21T19:40:18.318ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
66892024-11-21T19:40:18.318ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
66902024-11-21T19:40:18.320ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
66912024-11-21T19:40:18.320ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
66922024-11-21T19:40:18.322ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
66932024-11-21T19:40:18.322ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "316ab66d-3a9f-4ceb-bf19-ce17fe9b9cef", "content-length": "691", "date": "Thu, 21 Nov 2024 19:40:17 GMT"} })
66942024-11-21T19:40:18.322ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
66952024-11-21T19:40:18.322ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
66962024-11-21T19:40:18.322ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
66972024-11-21T19:40:18.323ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 489.997507ms iteration = 15 status = {}
66982024-11-21T19:40:18.324ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
66992024-11-21T19:40:18.325ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
67002024-11-21T19:40:18.328ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
67012024-11-21T19:40:18.328ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
67022024-11-21T19:40:18.328ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "d8bcd7e5-a1c4-49d1-8cb4-c7d2e1b820f8", "date": "Thu, 21 Nov 2024 19:40:17 GMT"} })
67032024-11-21T19:40:18.330ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 496.002227ms iteration = 15 status = {}
67042024-11-21T19:40:18.331ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
67052024-11-21T19:40:18.331ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "7d3de2a4-8bef-4a4e-93d4-7a66e178ea75", "content-length": "1", "date": "Thu, 21 Nov 2024 19:40:18 GMT"} })
67062024-11-21T19:40:18.334ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
67072024-11-21T19:40:18.334ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
67082024-11-21T19:40:18.337ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 503.358196ms iteration = 15 status = null
67092024-11-21T19:40:18.363ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
67102024-11-21T19:40:18.364ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
67112024-11-21T19:40:18.366ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
67122024-11-21T19:40:18.366ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
67132024-11-21T19:40:18.366ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
67142024-11-21T19:40:18.376ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
67152024-11-21T19:40:18.376ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "6524aac0-4954-4a6f-8f65-7e339a9cfcc2", "content-length": "134", "date": "Thu, 21 Nov 2024 19:40:18 GMT"} })
67162024-11-21T19:40:18.376ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
67172024-11-21T19:40:18.376ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
67182024-11-21T19:40:18.377ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
67192024-11-21T19:40:18.377ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
67202024-11-21T19:40:18.377ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
67212024-11-21T19:40:18.377ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
67222024-11-21T19:40:18.377ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
67232024-11-21T19:40:18.377ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
67242024-11-21T19:40:18.377ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
67252024-11-21T19:40:18.377ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
67262024-11-21T19:40:18.377ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
67272024-11-21T19:40:18.377ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
67282024-11-21T19:40:18.377ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
67292024-11-21T19:40:18.379ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
67302024-11-21T19:40:18.379ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "746710dc-af90-48ba-b413-e3cd3ac7e417", "content-length": "2", "date": "Thu, 21 Nov 2024 19:40:18 GMT"} })
67312024-11-21T19:40:18.379ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
67322024-11-21T19:40:18.379ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "3bc4a1b0-9fdb-4800-9c1d-ddb5133980d7", "content-length": "51", "date": "Thu, 21 Nov 2024 19:40:18 GMT"} })
67332024-11-21T19:40:18.379ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
67342024-11-21T19:40:18.379ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
67352024-11-21T19:40:18.474ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "b6bb022f-5446-4b1e-b9c1-1fd64b7438e1", "date": "Thu, 21 Nov 2024 19:40:17 GMT"} })
67362024-11-21T19:40:18.474ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
67372024-11-21T19:40:18.474ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
67382024-11-21T19:40:18.474ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "e33664e6-6e41-4b2b-800c-9381ac5256dd", "content-length": "682", "date": "Thu, 21 Nov 2024 19:40:17 GMT"} })
67392024-11-21T19:40:18.474ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
67402024-11-21T19:40:18.475ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
67412024-11-21T19:40:18.478ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
67422024-11-21T19:40:18.478ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
67432024-11-21T19:40:18.482ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
67442024-11-21T19:40:18.482ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
67452024-11-21T19:40:18.484ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
67462024-11-21T19:40:18.484ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
67472024-11-21T19:40:18.490ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 657.13084ms iteration = 15 status = {}
67482024-11-21T19:40:26.962ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 591 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 3f6290ba-45f8-4508-89c6-09c4dfe265fa response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
67492024-11-21T19:40:27.833ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 44 reason = Timeout
67502024-11-21T19:40:27.833ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
67512024-11-21T19:40:27.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
67522024-11-21T19:40:27.834ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 44 reason = Timeout
67532024-11-21T19:40:27.834ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
67542024-11-21T19:40:27.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
67552024-11-21T19:40:27.835ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
67562024-11-21T19:40:27.837ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
67572024-11-21T19:40:27.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
67582024-11-21T19:40:27.838ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
67592024-11-21T19:40:27.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
67602024-11-21T19:40:27.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
67612024-11-21T19:40:27.839ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
67622024-11-21T19:40:27.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
67632024-11-21T19:40:27.865ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 31.138196ms iteration = 44 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
67642024-11-21T19:40:27.867ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
67652024-11-21T19:40:27.870ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
67662024-11-21T19:40:27.870ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
67672024-11-21T19:40:27.870ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
67682024-11-21T19:40:27.881ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
67692024-11-21T19:40:27.881ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 47.9057ms iteration = 44 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
67702024-11-21T19:40:37.834ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 45 reason = Timeout
67712024-11-21T19:40:37.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
67722024-11-21T19:40:37.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
67732024-11-21T19:40:37.835ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 45 reason = Timeout
67742024-11-21T19:40:37.835ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
67752024-11-21T19:40:37.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
67762024-11-21T19:40:37.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
67772024-11-21T19:40:37.837ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
67782024-11-21T19:40:37.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
67792024-11-21T19:40:37.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
67802024-11-21T19:40:37.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
67812024-11-21T19:40:37.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
67822024-11-21T19:40:37.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
67832024-11-21T19:40:37.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
67842024-11-21T19:40:37.843ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
67852024-11-21T19:40:37.846ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
67862024-11-21T19:40:37.846ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
67872024-11-21T19:40:37.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
67882024-11-21T19:40:37.858ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
67892024-11-21T19:40:37.858ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 23.586536ms iteration = 45 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
67902024-11-21T19:40:37.868ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 33.745581ms iteration = 45 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
67912024-11-21T19:40:41.961ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 335 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 378082e3-6b14-4214-89d8-3bc28fdc1a3d response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
67922024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 16 reason = Timeout
67932024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
67942024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 46 reason = Timeout
67952024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 16 reason = Timeout
67962024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
67972024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
67982024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 16 reason = Timeout
67992024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 17 reason = Timeout
68002024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
68012024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 16 reason = Timeout
68022024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
68032024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
68042024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 16 reason = Timeout
68052024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 16 reason = Timeout
68062024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
68072024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 16 reason = Timeout
68082024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
68092024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 46 reason = Timeout
68102024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 16 reason = Timeout
68112024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
68122024-11-21T19:40:47.834ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
68132024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 16 reason = Timeout
68142024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
68152024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 16 reason = Timeout
68162024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
68172024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 16 reason = Timeout
68182024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
68192024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 16 reason = Timeout
68202024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 16 reason = Timeout
68212024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
68222024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
68232024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
68242024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 16 reason = Timeout
68252024-11-21T19:40:47.835ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
68262024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
68272024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
68282024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
68292024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
68302024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
68312024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
68322024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
68332024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
68342024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
68352024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
68362024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
68372024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
68382024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
68392024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
68402024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
68412024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
68422024-11-21T19:40:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
68432024-11-21T19:40:47.838ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.464226ms iteration = 16 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
68442024-11-21T19:40:47.838ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1190, tv_nsec: 712706086 } })
68452024-11-21T19:40:47.838ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1190, tv_nsec: 712919869 } })
68462024-11-21T19:40:47.838ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
68472024-11-21T19:40:47.838ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
68482024-11-21T19:40:47.839ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
68492024-11-21T19:40:47.841ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
68502024-11-21T19:40:47.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
68512024-11-21T19:40:47.842ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
68522024-11-21T19:40:47.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
68532024-11-21T19:40:47.843ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
68542024-11-21T19:40:47.843ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
68552024-11-21T19:40:47.843ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
68562024-11-21T19:40:47.845ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
68572024-11-21T19:40:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
68582024-11-21T19:40:47.848ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1190, tv_nsec: 722915652 } })
68592024-11-21T19:40:47.849ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
68602024-11-21T19:40:47.875ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
68612024-11-21T19:40:47.875ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
68622024-11-21T19:40:47.879ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
68632024-11-21T19:40:47.879ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
68642024-11-21T19:40:47.880ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
68652024-11-21T19:40:47.881ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
68662024-11-21T19:40:47.881ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
68672024-11-21T19:40:47.883ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
68682024-11-21T19:40:47.883ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
68692024-11-21T19:40:47.883ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
68702024-11-21T19:40:47.884ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
68712024-11-21T19:40:47.885ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
68722024-11-21T19:40:47.885ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 50.822529ms iteration = 16 status = {"errors":[],"records_set_to_done":[]}
68732024-11-21T19:40:47.886ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
68742024-11-21T19:40:47.887ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
68752024-11-21T19:40:47.887ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
68762024-11-21T19:40:47.887ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
68772024-11-21T19:40:47.887ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
68782024-11-21T19:40:47.888ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
68792024-11-21T19:40:47.888ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
68802024-11-21T19:40:47.888ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
68812024-11-21T19:40:47.888ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
68822024-11-21T19:40:47.888ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
68832024-11-21T19:40:47.890ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
68842024-11-21T19:40:47.890ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
68852024-11-21T19:40:47.890ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
68862024-11-21T19:40:47.890ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
68872024-11-21T19:40:47.890ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
68882024-11-21T19:40:47.890ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
68892024-11-21T19:40:47.892ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 57.997475ms iteration = 16 status = {"errors":[],"garbage_collect_requested":[]}
68902024-11-21T19:40:47.893ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
68912024-11-21T19:40:47.894ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
68922024-11-21T19:40:47.894ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
68932024-11-21T19:40:47.894ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
68942024-11-21T19:40:47.894ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 59.87665ms iteration = 16 status = 0
68952024-11-21T19:40:47.896ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
68962024-11-21T19:40:47.896ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
68972024-11-21T19:40:47.897ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
68982024-11-21T19:40:47.897ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
68992024-11-21T19:40:47.897ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
69002024-11-21T19:40:47.900ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
69012024-11-21T19:40:47.900ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
69022024-11-21T19:40:47.900ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
69032024-11-21T19:40:47.901ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
69042024-11-21T19:40:47.903ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
69052024-11-21T19:40:47.903ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
69062024-11-21T19:40:47.904ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
69072024-11-21T19:40:47.927ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
69082024-11-21T19:40:47.928ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
69092024-11-21T19:40:47.934ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 99.402269ms iteration = 46 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
69102024-11-21T19:40:47.935ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
69112024-11-21T19:40:47.937ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
69122024-11-21T19:40:47.937ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
69132024-11-21T19:40:47.937ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
69142024-11-21T19:40:47.937ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
69152024-11-21T19:40:47.938ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
69162024-11-21T19:40:47.938ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
69172024-11-21T19:40:47.938ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
69182024-11-21T19:40:47.947ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
69192024-11-21T19:40:47.948ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 113.804841ms iteration = 16 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
69202024-11-21T19:40:47.948ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
69212024-11-21T19:40:47.949ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
69222024-11-21T19:40:47.949ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
69232024-11-21T19:40:47.949ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 114.816354ms iteration = 16 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
69242024-11-21T19:40:47.950ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 115.333021ms iteration = 16 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
69252024-11-21T19:40:47.951ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
69262024-11-21T19:40:47.951ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
69272024-11-21T19:40:47.951ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
69282024-11-21T19:40:47.958ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 123.982527ms iteration = 16 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
69292024-11-21T19:40:47.967ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
69302024-11-21T19:40:47.967ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 132.970026ms iteration = 46 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
69312024-11-21T19:40:47.978ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
69322024-11-21T19:40:47.978ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
69332024-11-21T19:40:47.979ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "fa0ba5b3-acb5-432a-86f8-6d73cb34114d", "content-length": "26", "date": "Thu, 21 Nov 2024 19:40:47 GMT"} })
69342024-11-21T19:40:47.979ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
69352024-11-21T19:40:47.980ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
69362024-11-21T19:40:47.983ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
69372024-11-21T19:40:47.983ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
69382024-11-21T19:40:47.983ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 148.577315ms iteration = 16 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
69392024-11-21T19:40:48.033ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 198.247839ms iteration = 17 status = {"physical_disks_added":0}
69402024-11-21T19:40:48.043ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
69412024-11-21T19:40:48.043ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
69422024-11-21T19:40:48.043ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "b65b29ad-ea07-4641-aab5-463b1d4a9c17", "content-length": "26", "date": "Thu, 21 Nov 2024 19:40:47 GMT"} })
69432024-11-21T19:40:48.043ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
69442024-11-21T19:40:48.045ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
69452024-11-21T19:40:48.056ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
69462024-11-21T19:40:48.056ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
69472024-11-21T19:40:48.056ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "cc82202a-7e67-4abd-affe-f35b09078ef2", "content-length": "26", "date": "Thu, 21 Nov 2024 19:40:47 GMT"} })
69482024-11-21T19:40:48.056ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
69492024-11-21T19:40:48.058ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
69502024-11-21T19:40:48.060ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
69512024-11-21T19:40:48.061ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
69522024-11-21T19:40:48.123ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
69532024-11-21T19:40:48.123ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
69542024-11-21T19:40:48.123ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "ee1da398-3295-4b2b-b591-26cf695e1aa8", "content-length": "372", "date": "Thu, 21 Nov 2024 19:40:48 GMT"} })
69552024-11-21T19:40:48.123ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
69562024-11-21T19:40:48.124ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
69572024-11-21T19:40:48.154ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
69582024-11-21T19:40:48.156ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
69592024-11-21T19:40:48.156ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
69602024-11-21T19:40:48.156ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
69612024-11-21T19:40:48.157ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
69622024-11-21T19:40:48.157ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "55a6c169-a660-4a38-ac69-aab271b7c729", "content-length": "1", "date": "Thu, 21 Nov 2024 19:40:47 GMT"} })
69632024-11-21T19:40:48.158ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
69642024-11-21T19:40:48.159ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
69652024-11-21T19:40:48.159ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
69662024-11-21T19:40:48.160ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
69672024-11-21T19:40:48.160ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
69682024-11-21T19:40:48.160ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
69692024-11-21T19:40:48.161ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 326.588334ms iteration = 16 status = null
69702024-11-21T19:40:48.163ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 328.967096ms iteration = 16 status = {}
69712024-11-21T19:40:48.177ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
69722024-11-21T19:40:48.177ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
69732024-11-21T19:40:48.181ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "d24d4841-ea01-486c-8210-b08fa78fd07f", "date": "Thu, 21 Nov 2024 19:40:48 GMT"} })
69742024-11-21T19:40:48.181ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
69752024-11-21T19:40:48.182ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
69762024-11-21T19:40:48.184ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
69772024-11-21T19:40:48.184ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
69782024-11-21T19:40:48.188ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
69792024-11-21T19:40:48.188ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
69802024-11-21T19:40:48.192ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
69812024-11-21T19:40:48.192ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
69822024-11-21T19:40:48.193ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "2783746c-2200-4d9d-a680-a1f1511d0c83", "date": "Thu, 21 Nov 2024 19:40:48 GMT"} })
69832024-11-21T19:40:48.195ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 360.257713ms iteration = 16 status = {}
69842024-11-21T19:40:48.206ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
69852024-11-21T19:40:48.206ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
69862024-11-21T19:40:48.208ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
69872024-11-21T19:40:48.208ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
69882024-11-21T19:40:48.209ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
69892024-11-21T19:40:48.218ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
69902024-11-21T19:40:48.219ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "f7b07a06-8b81-4e27-92d8-fb7cfc97a33d", "content-length": "134", "date": "Thu, 21 Nov 2024 19:40:47 GMT"} })
69912024-11-21T19:40:48.219ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
69922024-11-21T19:40:48.219ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
69932024-11-21T19:40:48.219ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
69942024-11-21T19:40:48.219ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
69952024-11-21T19:40:48.219ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
69962024-11-21T19:40:48.219ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
69972024-11-21T19:40:48.219ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
69982024-11-21T19:40:48.219ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
69992024-11-21T19:40:48.219ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
70002024-11-21T19:40:48.219ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
70012024-11-21T19:40:48.219ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
70022024-11-21T19:40:48.219ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
70032024-11-21T19:40:48.220ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
70042024-11-21T19:40:48.221ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
70052024-11-21T19:40:48.222ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "70059695-ff44-4af8-ae6e-2b4381bfae2e", "content-length": "2", "date": "Thu, 21 Nov 2024 19:40:47 GMT"} })
70062024-11-21T19:40:48.222ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
70072024-11-21T19:40:48.222ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "5a9400c5-b00e-410c-92a3-1afe227d0d8f", "content-length": "51", "date": "Thu, 21 Nov 2024 19:40:48 GMT"} })
70082024-11-21T19:40:48.222ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
70092024-11-21T19:40:48.222ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
70102024-11-21T19:40:48.248ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
70112024-11-21T19:40:48.252ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "93a807df-d27e-464c-af60-4414522c0765", "content-length": "691", "date": "Thu, 21 Nov 2024 19:40:48 GMT"} })
70122024-11-21T19:40:48.252ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
70132024-11-21T19:40:48.252ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
70142024-11-21T19:40:48.252ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
70152024-11-21T19:40:48.254ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 419.644698ms iteration = 16 status = {}
70162024-11-21T19:40:48.311ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "a9c72e34-37ec-4543-9616-bbb74d02b6e5", "date": "Thu, 21 Nov 2024 19:40:47 GMT"} })
70172024-11-21T19:40:48.311ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
70182024-11-21T19:40:48.311ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
70192024-11-21T19:40:48.312ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "54b73caf-fe46-479f-a27b-7cd824e22438", "content-length": "682", "date": "Thu, 21 Nov 2024 19:40:48 GMT"} })
70202024-11-21T19:40:48.312ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
70212024-11-21T19:40:48.312ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
70222024-11-21T19:40:48.316ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
70232024-11-21T19:40:48.316ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
70242024-11-21T19:40:48.319ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
70252024-11-21T19:40:48.319ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
70262024-11-21T19:40:48.321ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
70272024-11-21T19:40:48.321ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
70282024-11-21T19:40:48.327ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 493.032998ms iteration = 16 status = {}
70292024-11-21T19:40:56.962ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 319 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = a5438536-8083-4a78-8fdc-3a580ff02458 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
70302024-11-21T19:40:57.834ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 47 reason = Timeout
70312024-11-21T19:40:57.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
70322024-11-21T19:40:57.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
70332024-11-21T19:40:57.835ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 47 reason = Timeout
70342024-11-21T19:40:57.835ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
70352024-11-21T19:40:57.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
70362024-11-21T19:40:57.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
70372024-11-21T19:40:57.837ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
70382024-11-21T19:40:57.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
70392024-11-21T19:40:57.839ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
70402024-11-21T19:40:57.839ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
70412024-11-21T19:40:57.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
70422024-11-21T19:40:57.841ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
70432024-11-21T19:40:57.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
70442024-11-21T19:40:57.866ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 30.896613ms iteration = 47 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
70452024-11-21T19:40:57.869ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
70462024-11-21T19:40:57.872ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
70472024-11-21T19:40:57.872ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
70482024-11-21T19:40:57.872ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
70492024-11-21T19:40:57.883ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
70502024-11-21T19:40:57.884ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 49.474411ms iteration = 47 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
70512024-11-21T19:41:07.834ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 48 reason = Timeout
70522024-11-21T19:41:07.834ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
70532024-11-21T19:41:07.834ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
70542024-11-21T19:41:07.835ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 48 reason = Timeout
70552024-11-21T19:41:07.835ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
70562024-11-21T19:41:07.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
70572024-11-21T19:41:07.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
70582024-11-21T19:41:07.837ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
70592024-11-21T19:41:07.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
70602024-11-21T19:41:07.839ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 3.920572ms iteration = 48 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
70612024-11-21T19:41:07.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
70622024-11-21T19:41:07.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
70632024-11-21T19:41:07.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
70642024-11-21T19:41:07.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
70652024-11-21T19:41:07.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
70662024-11-21T19:41:07.843ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
70672024-11-21T19:41:07.846ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
70682024-11-21T19:41:07.846ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
70692024-11-21T19:41:07.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
70702024-11-21T19:41:07.857ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
70712024-11-21T19:41:07.857ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 22.956566ms iteration = 48 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
70722024-11-21T19:41:11.963ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 299 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 3f77bfef-9526-4abc-8f33-769d6d78cd28 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
70732024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 17 reason = Timeout
70742024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 49 reason = Timeout
70752024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
70762024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
70772024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 10 reason = Timeout
70782024-11-21T19:41:17.835ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:83 reason = blueprint disabled target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb
70792024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 17.44\u{b5}s iteration = 10 status = {"enabled":false,"target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb"}
70802024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 9 reason = Timeout
70812024-11-21T19:41:17.835ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
70822024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 17 reason = Timeout
70832024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 9 reason = Timeout
70842024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
70852024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
70862024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
70872024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
70882024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 9 reason = Timeout
70892024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 18 reason = Timeout
70902024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 11 reason = Timeout
70912024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
70922024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 17 reason = Timeout
70932024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
70942024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 17 reason = Timeout
70952024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 9 reason = Timeout
70962024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 17 reason = Timeout
70972024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 17 reason = Timeout
70982024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 17 reason = Timeout
70992024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
71002024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 9 reason = Timeout
71012024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
71022024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
71032024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
71042024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 9 reason = Timeout
71052024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
71062024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 9 reason = Timeout
71072024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
71082024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 11 reason = Timeout
71092024-11-21T19:41:17.835ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-11-21 19:31:17.835409002 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
71102024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
71112024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
71122024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 17 reason = Timeout
71132024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
71142024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 17 reason = Timeout
71152024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 17 reason = Timeout
71162024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
71172024-11-21T19:41:17.835ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
71182024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
71192024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 9 reason = Timeout
71202024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 17 reason = Timeout
71212024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
71222024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
71232024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 17 reason = Timeout
71242024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
71252024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
71262024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = instance_reincarnation iteration = 9 reason = Timeout
71272024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 9 reason = Timeout
71282024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
71292024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
71302024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 17 reason = Timeout
71312024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
71322024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
71332024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 49 reason = Timeout
71342024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
71352024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 17 reason = Timeout
71362024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
71372024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
71382024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
71392024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
71402024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
71412024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
71422024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
71432024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
71442024-11-21T19:41:17.835ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0377cf96-baf3-4e09-a570-eb7c67037d4d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0d1b3b31-4591-4bcb-931d-e8e03365f0d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:1::1))) }], valid_until: Instant { tv_sec: 1220, tv_nsec: 709584624 } })
71452024-11-21T19:41:17.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
71462024-11-21T19:41:17.836ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("4e669ff4-6dde-4cd2-8491-623460b56d8d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 1220, tv_nsec: 709641215 } })
71472024-11-21T19:41:17.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
71482024-11-21T19:41:17.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
71492024-11-21T19:41:17.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
71502024-11-21T19:41:17.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
71512024-11-21T19:41:17.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
71522024-11-21T19:41:17.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
71532024-11-21T19:41:17.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
71542024-11-21T19:41:17.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
71552024-11-21T19:41:17.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
71562024-11-21T19:41:17.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
71572024-11-21T19:41:17.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
71582024-11-21T19:41:17.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
71592024-11-21T19:41:17.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
71602024-11-21T19:41:17.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
71612024-11-21T19:41:17.836ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } dns_group = internal
71622024-11-21T19:41:17.836ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 1.422469ms iteration = 9 status = {"addresses":["[fd00:1122:3344:3::1]:5353","[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353"]}
71632024-11-21T19:41:17.836ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
71642024-11-21T19:41:17.836ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 1.351068ms iteration = 9 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
71652024-11-21T19:41:17.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
71662024-11-21T19:41:17.838ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1220, tv_nsec: 712112928 } })
71672024-11-21T19:41:17.838ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
71682024-11-21T19:41:17.838ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
71692024-11-21T19:41:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
71702024-11-21T19:41:17.839ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
71712024-11-21T19:41:17.840ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
71722024-11-21T19:41:17.842ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 6.682819ms iteration = 17 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
71732024-11-21T19:41:17.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
71742024-11-21T19:41:17.842ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
71752024-11-21T19:41:17.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
71762024-11-21T19:41:17.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
71772024-11-21T19:41:17.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
71782024-11-21T19:41:17.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
71792024-11-21T19:41:17.844ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
71802024-11-21T19:41:17.844ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
71812024-11-21T19:41:17.844ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
71822024-11-21T19:41:17.846ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
71832024-11-21T19:41:17.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
71842024-11-21T19:41:17.847ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-11-21 19:31:17.835409002 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
71852024-11-21T19:41:17.847ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 12.216813ms iteration = 9 status = {"expiration":"2024-11-21T19:31:17.835409002Z","pruned":[]}
71862024-11-21T19:41:17.847ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
71872024-11-21T19:41:17.848ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
71882024-11-21T19:41:17.849ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1220, tv_nsec: 723194686 } })
71892024-11-21T19:41:17.849ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
71902024-11-21T19:41:17.850ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
71912024-11-21T19:41:17.850ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
71922024-11-21T19:41:17.850ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
71932024-11-21T19:41:17.851ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
71942024-11-21T19:41:17.851ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
71952024-11-21T19:41:17.852ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
71962024-11-21T19:41:17.854ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 18.717661ms iteration = 17 status = {"errors":[],"garbage_collect_requested":[]}
71972024-11-21T19:41:17.854ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 19.145236ms iteration = 9 status = {"errors":[],"found_port_ok":[]}
71982024-11-21T19:41:17.854ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
71992024-11-21T19:41:17.856ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
72002024-11-21T19:41:17.856ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
72012024-11-21T19:41:17.857ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
72022024-11-21T19:41:17.859ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
72032024-11-21T19:41:17.861ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
72042024-11-21T19:41:17.861ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
72052024-11-21T19:41:17.862ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1220, tv_nsec: 735799225 } })
72062024-11-21T19:41:17.862ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
72072024-11-21T19:41:17.863ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
72082024-11-21T19:41:17.865ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
72092024-11-21T19:41:17.867ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
72102024-11-21T19:41:17.867ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
72112024-11-21T19:41:17.867ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
72122024-11-21T19:41:17.867ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
72132024-11-21T19:41:17.869ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
72142024-11-21T19:41:17.871ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
72152024-11-21T19:41:17.871ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
72162024-11-21T19:41:17.871ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
72172024-11-21T19:41:17.871ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
72182024-11-21T19:41:17.871ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
72192024-11-21T19:41:17.871ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
72202024-11-21T19:41:17.871ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
72212024-11-21T19:41:17.877ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
72222024-11-21T19:41:17.877ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
72232024-11-21T19:41:17.877ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
72242024-11-21T19:41:17.877ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
72252024-11-21T19:41:17.877ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
72262024-11-21T19:41:17.877ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
72272024-11-21T19:41:17.890ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
72282024-11-21T19:41:17.895ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
72292024-11-21T19:41:17.895ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
72302024-11-21T19:41:17.895ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
72312024-11-21T19:41:17.895ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
72322024-11-21T19:41:17.897ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
72332024-11-21T19:41:17.897ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 62.225341ms iteration = 49 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
72342024-11-21T19:41:17.897ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
72352024-11-21T19:41:17.897ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
72362024-11-21T19:41:17.897ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
72372024-11-21T19:41:17.897ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
72382024-11-21T19:41:17.897ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
72392024-11-21T19:41:17.897ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
72402024-11-21T19:41:17.897ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
72412024-11-21T19:41:17.898ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
72422024-11-21T19:41:17.898ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
72432024-11-21T19:41:17.900ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
72442024-11-21T19:41:17.903ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
72452024-11-21T19:41:17.904ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
72462024-11-21T19:41:17.904ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
72472024-11-21T19:41:17.904ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
72482024-11-21T19:41:17.904ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
72492024-11-21T19:41:17.908ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 72.769922ms iteration = 17 status = {"errors":[],"records_set_to_done":[]}
72502024-11-21T19:41:17.908ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
72512024-11-21T19:41:17.908ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
72522024-11-21T19:41:17.908ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
72532024-11-21T19:41:17.908ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 73.409801ms iteration = 9 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
72542024-11-21T19:41:17.908ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0 total = 0
72552024-11-21T19:41:17.908ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:171 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0 vmms_found = 0
72562024-11-21T19:41:17.908ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 73.32494ms iteration = 9 status = {"errors":[],"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0,"vmms_found":0}
72572024-11-21T19:41:17.909ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
72582024-11-21T19:41:17.909ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 51 version = 1
72592024-11-21T19:41:17.909ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
72602024-11-21T19:41:17.909ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-11-21 19:33:31.797390 UTC dns_group = internal generation = 1
72612024-11-21T19:41:17.909ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 73.923708ms iteration = 9 status = {"generation":1}
72622024-11-21T19:41:17.909ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
72632024-11-21T19:41:17.909ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
72642024-11-21T19:41:17.910ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
72652024-11-21T19:41:17.910ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
72662024-11-21T19:41:17.912ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
72672024-11-21T19:41:17.912ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
72682024-11-21T19:41:17.912ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
72692024-11-21T19:41:17.913ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
72702024-11-21T19:41:17.914ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
72712024-11-21T19:41:17.914ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
72722024-11-21T19:41:17.914ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
72732024-11-21T19:41:17.914ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
72742024-11-21T19:41:17.916ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
72752024-11-21T19:41:17.916ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
72762024-11-21T19:41:17.916ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
72772024-11-21T19:41:17.916ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
72782024-11-21T19:41:17.916ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
72792024-11-21T19:41:17.916ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
72802024-11-21T19:41:17.916ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
72812024-11-21T19:41:17.916ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
72822024-11-21T19:41:17.916ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
72832024-11-21T19:41:17.917ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
72842024-11-21T19:41:17.917ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
72852024-11-21T19:41:17.917ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
72862024-11-21T19:41:17.918ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
72872024-11-21T19:41:17.919ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
72882024-11-21T19:41:17.920ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
72892024-11-21T19:41:17.920ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
72902024-11-21T19:41:17.920ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
72912024-11-21T19:41:17.920ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
72922024-11-21T19:41:17.924ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
72932024-11-21T19:41:17.924ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
72942024-11-21T19:41:17.924ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
72952024-11-21T19:41:17.924ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
72962024-11-21T19:41:17.925ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
72972024-11-21T19:41:17.926ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 91.24457ms iteration = 17 status = 0
72982024-11-21T19:41:17.927ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
72992024-11-21T19:41:17.927ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 91.514543ms iteration = 49 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
73002024-11-21T19:41:17.929ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
73012024-11-21T19:41:17.929ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
73022024-11-21T19:41:17.929ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
73032024-11-21T19:41:17.929ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
73042024-11-21T19:41:17.930ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
73052024-11-21T19:41:17.931ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
73062024-11-21T19:41:17.935ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
73072024-11-21T19:41:17.935ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
73082024-11-21T19:41:17.935ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
73092024-11-21T19:41:17.946ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 110.800091ms iteration = 17 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
73102024-11-21T19:41:17.955ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
73112024-11-21T19:41:17.955ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
73122024-11-21T19:41:17.964ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
73132024-11-21T19:41:17.964ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
73142024-11-21T19:41:17.964ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 128.441826ms iteration = 17 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
73152024-11-21T19:41:17.964ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 128.555888ms iteration = 17 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
73162024-11-21T19:41:17.964ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
73172024-11-21T19:41:17.964ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 129.288217ms iteration = 17 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
73182024-11-21T19:41:17.965ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
73192024-11-21T19:41:17.968ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
73202024-11-21T19:41:17.968ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
73212024-11-21T19:41:17.969ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
73222024-11-21T19:41:17.969ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
73232024-11-21T19:41:17.969ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
73242024-11-21T19:41:17.969ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
73252024-11-21T19:41:17.969ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "0322016a-177e-4a05-bdeb-9cbed0e3bb16", "content-length": "26", "date": "Thu, 21 Nov 2024 19:41:17 GMT"} })
73262024-11-21T19:41:17.969ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
73272024-11-21T19:41:17.971ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
73282024-11-21T19:41:17.971ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 136.096619ms iteration = 18 status = {"physical_disks_added":0}
73292024-11-21T19:41:17.975ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
73302024-11-21T19:41:17.975ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
73312024-11-21T19:41:18.000ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 164.452328ms iteration = 17 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
73322024-11-21T19:41:18.000ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-11-21T19:33:15.446964Z, time_modified: 2024-11-21T19:33:15.446964Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }) }
73332024-11-21T19:41:18.000ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 164.71493ms iteration = 9 status = {"by_dns_name":{"default-silo.sys.oxide.test":{"silo_id":"001de000-5110-4000-8000-000000000000","tls_certs":[]},"recovery.sys.oxide.test":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]}},"default_endpoint":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]},"warnings":["silo 001de000-5110-4000-8000-000000000000 with DNS name \\"default-silo.sys.oxide.test\\" has no usable certificates"]}
73342024-11-21T19:41:18.002ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
73352024-11-21T19:41:18.005ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
73362024-11-21T19:41:18.005ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 0 version = 2
73372024-11-21T19:41:18.005ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
73382024-11-21T19:41:18.005ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
73392024-11-21T19:41:18.007ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
73402024-11-21T19:41:18.011ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
73412024-11-21T19:41:18.011ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
73422024-11-21T19:41:18.011ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
73432024-11-21T19:41:18.014ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
73442024-11-21T19:41:18.014ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
73452024-11-21T19:41:18.015ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "5fd16cb0-5ffc-4acc-854a-9e6702c1a285", "content-length": "26", "date": "Thu, 21 Nov 2024 19:41:17 GMT"} })
73462024-11-21T19:41:18.015ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
73472024-11-21T19:41:18.016ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 1 version = 2
73482024-11-21T19:41:18.016ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
73492024-11-21T19:41:18.016ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-11-21 19:33:32.758954 UTC dns_group = external generation = 2
73502024-11-21T19:41:18.016ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 180.962258ms iteration = 9 status = {"generation":2}
73512024-11-21T19:41:18.017ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
73522024-11-21T19:41:18.023ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
73532024-11-21T19:41:18.024ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
73542024-11-21T19:41:18.024ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "2b63586c-e99f-4855-a1eb-e50723a99da6", "content-length": "26", "date": "Thu, 21 Nov 2024 19:41:17 GMT"} })
73552024-11-21T19:41:18.024ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
73562024-11-21T19:41:18.025ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
73572024-11-21T19:41:18.044ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
73582024-11-21T19:41:18.045ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
73592024-11-21T19:41:18.083ZWARNnexus (BackgroundTasks): instance reincarnation completed with errors background_task = instance_reincarnation file = nexus/src/app/background/tasks/instance_reincarnation.rs:103 instances_changed_state = 0 instances_found = 0 instances_reincarnated = 0 query_errors = 0 restart_errors = 0
73602024-11-21T19:41:18.083ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_reincarnation elapsed = 248.073734ms iteration = 9 status = {"changed_state":[],"disabled":false,"errors":[],"instances_found":{"Failed":0,"SagaUnwound":0},"instances_reincarnated":[],"restart_errors":[]}
73612024-11-21T19:41:18.088ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
73622024-11-21T19:41:18.090ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
73632024-11-21T19:41:18.090ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
73642024-11-21T19:41:18.091ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
73652024-11-21T19:41:18.092ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
73662024-11-21T19:41:18.094ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
73672024-11-21T19:41:18.094ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
73682024-11-21T19:41:18.094ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
73692024-11-21T19:41:18.145ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
73702024-11-21T19:41:18.145ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::8]:5353/config", status: 204, headers: {"x-request-id": "db18200b-e6f8-497f-8e9b-3ba6dc6d3562", "date": "Thu, 21 Nov 2024 19:41:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
73712024-11-21T19:41:18.145ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
73722024-11-21T19:41:18.147ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::9]:5353/config", status: 204, headers: {"x-request-id": "05fe87bf-ec28-49a7-8705-f8e48a94ee43", "date": "Thu, 21 Nov 2024 19:41:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
73732024-11-21T19:41:18.147ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
73742024-11-21T19:41:18.148ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
73752024-11-21T19:41:18.148ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 313.162715ms iteration = 11 status = {"generation":2,"server_results":{"[fd00:1122:3344:101::8]:5353":{"Ok":null},"[fd00:1122:3344:101::9]:5353":{"Ok":null}}}
73762024-11-21T19:41:18.182ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
73772024-11-21T19:41:18.227ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 392.583206ms iteration = 17 status = {}
73782024-11-21T19:41:18.277ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
73792024-11-21T19:41:18.277ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "bdd4443b-d7b5-4da5-bfe9-d3d73ccc1468", "content-length": "1", "date": "Thu, 21 Nov 2024 19:41:18 GMT"} })
73802024-11-21T19:41:18.280ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
73812024-11-21T19:41:18.280ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
73822024-11-21T19:41:18.295ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
73832024-11-21T19:41:18.296ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "f810ae69-e2d9-405b-b4d9-656a39e03e99", "content-length": "691", "date": "Thu, 21 Nov 2024 19:41:18 GMT"} })
73842024-11-21T19:41:18.296ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
73852024-11-21T19:41:18.296ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
73862024-11-21T19:41:18.296ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
73872024-11-21T19:41:18.298ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 462.805444ms iteration = 17 status = {}
73882024-11-21T19:41:18.310ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
73892024-11-21T19:41:18.311ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:1::1]:5353/config", status: 204, headers: {"x-request-id": "547f3525-3cd4-4483-8d81-affa0d369892", "date": "Thu, 21 Nov 2024 19:41:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
73902024-11-21T19:41:18.311ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
73912024-11-21T19:41:18.312ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:2::1]:5353/config", status: 204, headers: {"x-request-id": "025ffbb0-da22-4a34-be74-e47f13aa6102", "date": "Thu, 21 Nov 2024 19:41:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
73922024-11-21T19:41:18.312ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
73932024-11-21T19:41:18.314ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:3::1]:5353/config", status: 204, headers: {"x-request-id": "f862f980-08be-47f4-bb1b-2995c6f74a3c", "date": "Thu, 21 Nov 2024 19:41:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
73942024-11-21T19:41:18.314ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
73952024-11-21T19:41:18.315ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
73962024-11-21T19:41:18.315ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 480.46298ms iteration = 11 status = {"generation":1,"server_results":{"[fd00:1122:3344:1::1]:5353":{"Ok":null},"[fd00:1122:3344:2::1]:5353":{"Ok":null},"[fd00:1122:3344:3::1]:5353":{"Ok":null}}}
73972024-11-21T19:41:18.318ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
73982024-11-21T19:41:18.318ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
73992024-11-21T19:41:18.319ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "28426252-144c-49bf-a7fd-2199533b8600", "content-length": "372", "date": "Thu, 21 Nov 2024 19:41:18 GMT"} })
74002024-11-21T19:41:18.319ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
74012024-11-21T19:41:18.319ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
74022024-11-21T19:41:18.321ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 485.700579ms iteration = 17 status = null
74032024-11-21T19:41:18.332ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
74042024-11-21T19:41:18.332ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
74052024-11-21T19:41:18.333ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "6b0c9147-18d8-4cbd-8603-b9bc7ec2af2c", "date": "Thu, 21 Nov 2024 19:41:18 GMT"} })
74062024-11-21T19:41:18.333ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
74072024-11-21T19:41:18.333ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
74082024-11-21T19:41:18.335ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
74092024-11-21T19:41:18.335ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
74102024-11-21T19:41:18.338ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
74112024-11-21T19:41:18.339ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
74122024-11-21T19:41:18.341ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
74132024-11-21T19:41:18.341ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
74142024-11-21T19:41:18.341ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "3d816078-c375-4ea7-9f10-04773f3bf75b", "date": "Thu, 21 Nov 2024 19:41:18 GMT"} })
74152024-11-21T19:41:18.343ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 507.986428ms iteration = 17 status = {}
74162024-11-21T19:41:18.381ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
74172024-11-21T19:41:18.381ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
74182024-11-21T19:41:18.383ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
74192024-11-21T19:41:18.383ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
74202024-11-21T19:41:18.383ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
74212024-11-21T19:41:18.393ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
74222024-11-21T19:41:18.393ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "e28466a8-3573-4d91-bbc5-681f81c6d6a1", "content-length": "134", "date": "Thu, 21 Nov 2024 19:41:18 GMT"} })
74232024-11-21T19:41:18.393ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
74242024-11-21T19:41:18.393ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
74252024-11-21T19:41:18.393ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
74262024-11-21T19:41:18.393ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
74272024-11-21T19:41:18.393ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
74282024-11-21T19:41:18.393ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
74292024-11-21T19:41:18.393ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
74302024-11-21T19:41:18.393ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
74312024-11-21T19:41:18.393ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
74322024-11-21T19:41:18.393ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
74332024-11-21T19:41:18.394ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
74342024-11-21T19:41:18.394ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
74352024-11-21T19:41:18.394ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
74362024-11-21T19:41:18.396ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
74372024-11-21T19:41:18.396ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "45522737-62f8-48d2-840d-ad3c8ce67be1", "content-length": "2", "date": "Thu, 21 Nov 2024 19:41:18 GMT"} })
74382024-11-21T19:41:18.396ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
74392024-11-21T19:41:18.396ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "954a182f-7138-4cb0-82c2-8a9ebacb209c", "content-length": "51", "date": "Thu, 21 Nov 2024 19:41:18 GMT"} })
74402024-11-21T19:41:18.396ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
74412024-11-21T19:41:18.396ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
74422024-11-21T19:41:18.489ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "7ecf7ba2-cc72-44d5-bb00-604172cd2a3e", "date": "Thu, 21 Nov 2024 19:41:18 GMT"} })
74432024-11-21T19:41:18.490ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
74442024-11-21T19:41:18.490ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
74452024-11-21T19:41:18.490ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "2f726b27-d3c9-4101-96e0-468aae7d7596", "content-length": "682", "date": "Thu, 21 Nov 2024 19:41:18 GMT"} })
74462024-11-21T19:41:18.490ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
74472024-11-21T19:41:18.490ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
74482024-11-21T19:41:18.493ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
74492024-11-21T19:41:18.494ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
74502024-11-21T19:41:18.497ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
74512024-11-21T19:41:18.497ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
74522024-11-21T19:41:18.499ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
74532024-11-21T19:41:18.499ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
74542024-11-21T19:41:18.506ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 671.234938ms iteration = 17 status = {}
74552024-11-21T19:41:26.963ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 681 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 44930ada-0864-4a48-be57-0af5390178f6 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
74562024-11-21T19:41:27.835ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 50 reason = Timeout
74572024-11-21T19:41:27.835ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
74582024-11-21T19:41:27.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
74592024-11-21T19:41:27.836ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 50 reason = Timeout
74602024-11-21T19:41:27.836ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
74612024-11-21T19:41:27.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
74622024-11-21T19:41:27.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
74632024-11-21T19:41:27.838ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
74642024-11-21T19:41:27.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
74652024-11-21T19:41:27.839ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 3.640509ms iteration = 50 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
74662024-11-21T19:41:27.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
74672024-11-21T19:41:27.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
74682024-11-21T19:41:27.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
74692024-11-21T19:41:27.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
74702024-11-21T19:41:27.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
74712024-11-21T19:41:27.844ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
74722024-11-21T19:41:27.846ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
74732024-11-21T19:41:27.846ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
74742024-11-21T19:41:27.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
74752024-11-21T19:41:27.854ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
74762024-11-21T19:41:27.854ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 19.636422ms iteration = 50 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
74772024-11-21T19:41:37.836ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 51 reason = Timeout
74782024-11-21T19:41:37.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
74792024-11-21T19:41:37.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
74802024-11-21T19:41:37.836ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 51 reason = Timeout
74812024-11-21T19:41:37.836ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
74822024-11-21T19:41:37.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
74832024-11-21T19:41:37.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
74842024-11-21T19:41:37.838ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
74852024-11-21T19:41:37.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
74862024-11-21T19:41:37.841ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
74872024-11-21T19:41:37.841ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
74882024-11-21T19:41:37.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
74892024-11-21T19:41:37.843ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
74902024-11-21T19:41:37.843ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
74912024-11-21T19:41:37.845ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
74922024-11-21T19:41:37.848ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
74932024-11-21T19:41:37.848ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
74942024-11-21T19:41:37.848ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
74952024-11-21T19:41:37.860ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
74962024-11-21T19:41:37.860ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 24.875472ms iteration = 51 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
74972024-11-21T19:41:37.866ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 29.89061ms iteration = 51 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
74982024-11-21T19:41:41.964ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 347 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = ea380aee-4af9-4814-95d9-bb8d19d04d2b response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
74992024-11-21T19:41:46.956ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = 3b875aea-6db5-4129-bd9f-a87c31afcaf0 roles = RoleSet { roles: {} } uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
75002024-11-21T19:41:46.956ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = 3b875aea-6db5-4129-bd9f-a87c31afcaf0 resource = Database result = Ok(()) uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
75012024-11-21T19:41:46.959ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 3468 local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = 3b875aea-6db5-4129-bd9f-a87c31afcaf0 response_code = 200 uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
75022024-11-21T19:41:46.960ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = ad2a112c-1347-40ff-9dea-f7bc931de1d7 roles = RoleSet { roles: {} } uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
75032024-11-21T19:41:46.960ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = ad2a112c-1347-40ff-9dea-f7bc931de1d7 resource = Database result = Ok(()) uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
75042024-11-21T19:41:46.963ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 3006 local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = ad2a112c-1347-40ff-9dea-f7bc931de1d7 response_code = 200 uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
75052024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 18 reason = Timeout
75062024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
75072024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 52 reason = Timeout
75082024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 18 reason = Timeout
75092024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
75102024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
75112024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 18 reason = Timeout
75122024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 18 reason = Timeout
75132024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
75142024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
75152024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 18 reason = Timeout
75162024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 19 reason = Timeout
75172024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
75182024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
75192024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 18 reason = Timeout
75202024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 18 reason = Timeout
75212024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
75222024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
75232024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
75242024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
75252024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
75262024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
75272024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
75282024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
75292024-11-21T19:41:47.835ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
75302024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 52 reason = Timeout
75312024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 18 reason = Timeout
75322024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
75332024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
75342024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 18 reason = Timeout
75352024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
75362024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 18 reason = Timeout
75372024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 18 reason = Timeout
75382024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
75392024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
75402024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
75412024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 18 reason = Timeout
75422024-11-21T19:41:47.836ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
75432024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
75442024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 18 reason = Timeout
75452024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
75462024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 18 reason = Timeout
75472024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
75482024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
75492024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
75502024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
75512024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
75522024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
75532024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
75542024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
75552024-11-21T19:41:47.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
75562024-11-21T19:41:47.839ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1250, tv_nsec: 712392432 } })
75572024-11-21T19:41:47.839ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
75582024-11-21T19:41:47.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
75592024-11-21T19:41:47.840ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
75602024-11-21T19:41:47.840ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1250, tv_nsec: 713479716 } })
75612024-11-21T19:41:47.840ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 5.189709ms iteration = 18 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
75622024-11-21T19:41:47.840ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
75632024-11-21T19:41:47.842ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
75642024-11-21T19:41:47.843ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
75652024-11-21T19:41:47.843ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
75662024-11-21T19:41:47.843ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
75672024-11-21T19:41:47.843ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
75682024-11-21T19:41:47.843ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
75692024-11-21T19:41:47.843ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
75702024-11-21T19:41:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
75712024-11-21T19:41:47.845ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
75722024-11-21T19:41:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
75732024-11-21T19:41:47.847ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
75742024-11-21T19:41:47.848ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
75752024-11-21T19:41:47.850ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
75762024-11-21T19:41:47.850ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
75772024-11-21T19:41:47.851ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
75782024-11-21T19:41:47.851ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
75792024-11-21T19:41:47.851ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
75802024-11-21T19:41:47.851ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
75812024-11-21T19:41:47.851ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
75822024-11-21T19:41:47.851ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
75832024-11-21T19:41:47.853ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
75842024-11-21T19:41:47.853ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
75852024-11-21T19:41:47.854ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
75862024-11-21T19:41:47.854ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
75872024-11-21T19:41:47.855ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1250, tv_nsec: 728143182 } })
75882024-11-21T19:41:47.855ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
75892024-11-21T19:41:47.855ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
75902024-11-21T19:41:47.856ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
75912024-11-21T19:41:47.858ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
75922024-11-21T19:41:47.858ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
75932024-11-21T19:41:47.858ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
75942024-11-21T19:41:47.859ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
75952024-11-21T19:41:47.859ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
75962024-11-21T19:41:47.859ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
75972024-11-21T19:41:47.876ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
75982024-11-21T19:41:47.876ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 41.136909ms iteration = 52 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
75992024-11-21T19:41:47.886ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
76002024-11-21T19:41:47.887ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
76012024-11-21T19:41:47.889ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
76022024-11-21T19:41:47.889ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
76032024-11-21T19:41:47.889ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
76042024-11-21T19:41:47.889ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 54.421208ms iteration = 18 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
76052024-11-21T19:41:47.890ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 53.628926ms iteration = 18 status = {"errors":[],"garbage_collect_requested":[]}
76062024-11-21T19:41:47.892ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
76072024-11-21T19:41:47.892ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
76082024-11-21T19:41:47.892ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
76092024-11-21T19:41:47.894ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
76102024-11-21T19:41:47.895ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
76112024-11-21T19:41:47.895ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
76122024-11-21T19:41:47.895ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
76132024-11-21T19:41:47.896ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 60.387876ms iteration = 52 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
76142024-11-21T19:41:47.900ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
76152024-11-21T19:41:47.900ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
76162024-11-21T19:41:47.900ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
76172024-11-21T19:41:47.900ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
76182024-11-21T19:41:47.901ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
76192024-11-21T19:41:47.902ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
76202024-11-21T19:41:47.902ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
76212024-11-21T19:41:47.902ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
76222024-11-21T19:41:47.903ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
76232024-11-21T19:41:47.903ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
76242024-11-21T19:41:47.903ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 66.670031ms iteration = 18 status = {"errors":[],"records_set_to_done":[]}
76252024-11-21T19:41:47.903ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
76262024-11-21T19:41:47.904ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
76272024-11-21T19:41:47.904ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
76282024-11-21T19:41:47.904ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
76292024-11-21T19:41:47.910ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
76302024-11-21T19:41:47.910ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 74.021099ms iteration = 18 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
76312024-11-21T19:41:47.911ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
76322024-11-21T19:41:47.911ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
76332024-11-21T19:41:47.911ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "48a3d87a-20a2-4b57-b6b3-a2e1b98d4201", "content-length": "26", "date": "Thu, 21 Nov 2024 19:41:47 GMT"} })
76342024-11-21T19:41:47.911ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
76352024-11-21T19:41:47.912ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
76362024-11-21T19:41:47.912ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
76372024-11-21T19:41:47.913ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "e1e665a7-e9f0-4196-aa51-014c5b2796e0", "content-length": "26", "date": "Thu, 21 Nov 2024 19:41:47 GMT"} })
76382024-11-21T19:41:47.913ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
76392024-11-21T19:41:47.914ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
76402024-11-21T19:41:47.914ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
76412024-11-21T19:41:47.919ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
76422024-11-21T19:41:47.919ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
76432024-11-21T19:41:47.922ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
76442024-11-21T19:41:47.922ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
76452024-11-21T19:41:47.924ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
76462024-11-21T19:41:47.926ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
76472024-11-21T19:41:47.927ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
76482024-11-21T19:41:47.927ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
76492024-11-21T19:41:47.929ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 93.770523ms iteration = 18 status = 0
76502024-11-21T19:41:47.929ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
76512024-11-21T19:41:47.930ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
76522024-11-21T19:41:47.933ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 96.610471ms iteration = 18 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
76532024-11-21T19:41:47.934ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
76542024-11-21T19:41:47.934ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
76552024-11-21T19:41:47.934ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 98.290653ms iteration = 18 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
76562024-11-21T19:41:47.949ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
76572024-11-21T19:41:47.949ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
76582024-11-21T19:41:47.950ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "945ed732-94ce-4c46-a482-f03db16e5204", "content-length": "26", "date": "Thu, 21 Nov 2024 19:41:47 GMT"} })
76592024-11-21T19:41:47.950ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
76602024-11-21T19:41:47.951ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
76612024-11-21T19:41:47.967ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 131.008971ms iteration = 18 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
76622024-11-21T19:41:47.975ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
76632024-11-21T19:41:47.975ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
76642024-11-21T19:41:47.977ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
76652024-11-21T19:41:47.979ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
76662024-11-21T19:41:47.979ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
76672024-11-21T19:41:47.979ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
76682024-11-21T19:41:47.998ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 163.037168ms iteration = 19 status = {"physical_disks_added":0}
76692024-11-21T19:41:48.013ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
76702024-11-21T19:41:48.013ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "37a490fd-e40d-4933-830f-92c75f95b33b", "content-length": "1", "date": "Thu, 21 Nov 2024 19:41:47 GMT"} })
76712024-11-21T19:41:48.015ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
76722024-11-21T19:41:48.016ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
76732024-11-21T19:41:48.020ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 185.052532ms iteration = 18 status = {}
76742024-11-21T19:41:48.048ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
76752024-11-21T19:41:48.048ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
76762024-11-21T19:41:48.049ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
76772024-11-21T19:41:48.050ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "3c7a0edf-9227-489a-9773-9e971d99af52", "content-length": "372", "date": "Thu, 21 Nov 2024 19:41:48 GMT"} })
76782024-11-21T19:41:48.050ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
76792024-11-21T19:41:48.050ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
76802024-11-21T19:41:48.050ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 215.438049ms iteration = 18 status = null
76812024-11-21T19:41:48.051ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
76822024-11-21T19:41:48.052ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
76832024-11-21T19:41:48.052ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
76842024-11-21T19:41:48.065ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
76852024-11-21T19:41:48.065ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
76862024-11-21T19:41:48.065ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "a42a2c42-0213-4476-bd3a-90548f50dc38", "date": "Thu, 21 Nov 2024 19:41:48 GMT"} })
76872024-11-21T19:41:48.065ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
76882024-11-21T19:41:48.066ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
76892024-11-21T19:41:48.067ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
76902024-11-21T19:41:48.068ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
76912024-11-21T19:41:48.071ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
76922024-11-21T19:41:48.071ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
76932024-11-21T19:41:48.074ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
76942024-11-21T19:41:48.074ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
76952024-11-21T19:41:48.077ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "52471154-ca24-4122-a65d-f1542c5101e0", "date": "Thu, 21 Nov 2024 19:41:48 GMT"} })
76962024-11-21T19:41:48.080ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 243.608505ms iteration = 18 status = {}
76972024-11-21T19:41:48.115ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
76982024-11-21T19:41:48.116ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "93877451-ffaf-47f1-af82-e34ab1603d9f", "content-length": "691", "date": "Thu, 21 Nov 2024 19:41:47 GMT"} })
76992024-11-21T19:41:48.116ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
77002024-11-21T19:41:48.116ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
77012024-11-21T19:41:48.116ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
77022024-11-21T19:41:48.118ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 282.845659ms iteration = 18 status = {}
77032024-11-21T19:41:48.136ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
77042024-11-21T19:41:48.137ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
77052024-11-21T19:41:48.140ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
77062024-11-21T19:41:48.140ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
77072024-11-21T19:41:48.141ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
77082024-11-21T19:41:48.148ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
77092024-11-21T19:41:48.148ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "62a3f812-2263-4995-bac2-6fff8378541e", "content-length": "134", "date": "Thu, 21 Nov 2024 19:41:47 GMT"} })
77102024-11-21T19:41:48.148ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
77112024-11-21T19:41:48.148ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
77122024-11-21T19:41:48.149ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
77132024-11-21T19:41:48.149ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
77142024-11-21T19:41:48.149ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
77152024-11-21T19:41:48.149ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
77162024-11-21T19:41:48.149ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
77172024-11-21T19:41:48.149ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
77182024-11-21T19:41:48.149ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
77192024-11-21T19:41:48.149ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
77202024-11-21T19:41:48.149ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
77212024-11-21T19:41:48.149ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
77222024-11-21T19:41:48.149ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
77232024-11-21T19:41:48.151ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
77242024-11-21T19:41:48.152ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "33bff51e-edc6-4605-a85d-75d83310ba91", "content-length": "2", "date": "Thu, 21 Nov 2024 19:41:47 GMT"} })
77252024-11-21T19:41:48.152ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
77262024-11-21T19:41:48.152ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "63c44f18-f048-4726-81c5-65eecb494df0", "content-length": "51", "date": "Thu, 21 Nov 2024 19:41:47 GMT"} })
77272024-11-21T19:41:48.152ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
77282024-11-21T19:41:48.152ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
77292024-11-21T19:41:48.255ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "7b9f3a7a-41ff-4371-bb47-11de767fc415", "date": "Thu, 21 Nov 2024 19:41:47 GMT"} })
77302024-11-21T19:41:48.255ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
77312024-11-21T19:41:48.255ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
77322024-11-21T19:41:48.256ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "980ca5cf-7899-4673-9b7d-a61cc9a7792e", "content-length": "682", "date": "Thu, 21 Nov 2024 19:41:47 GMT"} })
77332024-11-21T19:41:48.256ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
77342024-11-21T19:41:48.256ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
77352024-11-21T19:41:48.260ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
77362024-11-21T19:41:48.260ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
77372024-11-21T19:41:48.263ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
77382024-11-21T19:41:48.264ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
77392024-11-21T19:41:48.265ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
77402024-11-21T19:41:48.265ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
77412024-11-21T19:41:48.273ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 437.666177ms iteration = 18 status = {}
77422024-11-21T19:41:48.511ZDEBGnexus (producer-registration-task): registering / renewing oximeter producer lease with Nexus
77432024-11-21T19:41:48.511ZDEBGnexus (producer-registration-task): using nexus address for registration addr = [fd00:1122:3344:101::c]:12221
77442024-11-21T19:41:48.542ZDEBGnexus (producer-registration-task): client request body = Some(Body) method = POST uri = http://[fd00:1122:3344:101::c]:12221/metrics/producers
77452024-11-21T19:41:48.542ZINFOnexus (dropshot_internal): accepted connection file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023 local_addr = [fd00:1122:3344:101::c]:12221 remote_addr = [fd00:1122:3344:101::c]:51391
77462024-11-21T19:41:48.542ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:51391 req_id = b3498fe9-1023-4c32-9851-af995cfd5cf8 roles = RoleSet { roles: {} } uri = /metrics/producers
77472024-11-21T19:41:48.543ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:51391 req_id = b3498fe9-1023-4c32-9851-af995cfd5cf8 resource = Database result = Ok(()) uri = /metrics/producers
77482024-11-21T19:41:48.611ZINFOnexus: assigned collector to new producer collector_id = 5fb99117-c685-44d5-bec2-c58adb63b619 file = nexus/src/app/oximeter.rs:76 producer_id = 915e11b8-d7cb-4730-9906-7980540813dc
77492024-11-21T19:41:48.611ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 68423 local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:51391 req_id = b3498fe9-1023-4c32-9851-af995cfd5cf8 response_code = 201 uri = /metrics/producers
77502024-11-21T19:41:48.611ZDEBGnexus (producer-registration-task): client response result = Ok(Response { url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", status: 201, headers: {"content-type": "application/json", "x-request-id": "b3498fe9-1023-4c32-9851-af995cfd5cf8", "content-length": "41", "date": "Thu, 21 Nov 2024 19:41:48 GMT"} })
77512024-11-21T19:41:48.613ZDEBGnexus (producer-registration-task): registered with nexus successfully
77522024-11-21T19:41:48.613ZDEBGnexus (producer-registration-task): pausing until time to renew lease lease_duration = 600s wait_period = 150s
77532024-11-21T19:41:56.964ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 329 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 41a38a0d-ea0f-4210-a13e-8e9eb6aed0df response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
77542024-11-21T19:41:57.835ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 53 reason = Timeout
77552024-11-21T19:41:57.835ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
77562024-11-21T19:41:57.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
77572024-11-21T19:41:57.836ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 53 reason = Timeout
77582024-11-21T19:41:57.836ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
77592024-11-21T19:41:57.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
77602024-11-21T19:41:57.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
77612024-11-21T19:41:57.838ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
77622024-11-21T19:41:57.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
77632024-11-21T19:41:57.840ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 3.509437ms iteration = 53 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
77642024-11-21T19:41:57.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
77652024-11-21T19:41:57.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
77662024-11-21T19:41:57.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
77672024-11-21T19:41:57.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
77682024-11-21T19:41:57.843ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
77692024-11-21T19:41:57.844ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
77702024-11-21T19:41:57.847ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
77712024-11-21T19:41:57.847ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
77722024-11-21T19:41:57.847ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
77732024-11-21T19:41:57.858ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
77742024-11-21T19:41:57.858ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 22.311338ms iteration = 53 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
77752024-11-21T19:42:07.836ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 54 reason = Timeout
77762024-11-21T19:42:07.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
77772024-11-21T19:42:07.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
77782024-11-21T19:42:07.836ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 54 reason = Timeout
77792024-11-21T19:42:07.836ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
77802024-11-21T19:42:07.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
77812024-11-21T19:42:07.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
77822024-11-21T19:42:07.838ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
77832024-11-21T19:42:07.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
77842024-11-21T19:42:07.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
77852024-11-21T19:42:07.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
77862024-11-21T19:42:07.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
77872024-11-21T19:42:07.842ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 5.775717ms iteration = 54 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
77882024-11-21T19:42:07.845ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
77892024-11-21T19:42:07.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
77902024-11-21T19:42:07.847ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
77912024-11-21T19:42:07.849ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
77922024-11-21T19:42:07.849ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
77932024-11-21T19:42:07.849ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
77942024-11-21T19:42:07.862ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
77952024-11-21T19:42:07.862ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 26.031207ms iteration = 54 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
77962024-11-21T19:42:11.964ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 324 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 1a12c1d5-56c6-47e4-89fe-bb5c8956fbae response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
77972024-11-21T19:42:16.957ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = 48ca5889-37cf-417e-8be4-d5dffe1bf705 roles = RoleSet { roles: {} } uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
77982024-11-21T19:42:16.957ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = 48ca5889-37cf-417e-8be4-d5dffe1bf705 resource = Database result = Ok(()) uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
77992024-11-21T19:42:16.960ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 3287 local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = 48ca5889-37cf-417e-8be4-d5dffe1bf705 response_code = 200 uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
78002024-11-21T19:42:16.960ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = b7edd055-9c05-4188-b145-9dc1acefbfd7 roles = RoleSet { roles: {} } uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
78012024-11-21T19:42:16.961ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = b7edd055-9c05-4188-b145-9dc1acefbfd7 resource = Database result = Ok(()) uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
78022024-11-21T19:42:16.963ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 2629 local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:56016 req_id = b7edd055-9c05-4188-b145-9dc1acefbfd7 response_code = 200 uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
78032024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 19 reason = Timeout
78042024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
78052024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 10 reason = Timeout
78062024-11-21T19:42:17.836ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
78072024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
78082024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 11 reason = Timeout
78092024-11-21T19:42:17.836ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:83 reason = blueprint disabled target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb
78102024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 20.15\u{b5}s iteration = 11 status = {"enabled":false,"target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb"}
78112024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 19 reason = Timeout
78122024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 10 reason = Timeout
78132024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 19 reason = Timeout
78142024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
78152024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
78162024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
78172024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
78182024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 10 reason = Timeout
78192024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 19 reason = Timeout
78202024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 10 reason = Timeout
78212024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
78222024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 20 reason = Timeout
78232024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 10 reason = Timeout
78242024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 12 reason = Timeout
78252024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
78262024-11-21T19:42:17.836ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-11-21 19:32:17.836688804 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
78272024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
78282024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 10 reason = Timeout
78292024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
78302024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
78312024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 19 reason = Timeout
78322024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
78332024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 10 reason = Timeout
78342024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 19 reason = Timeout
78352024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
78362024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = crdb_node_id_collector iteration = 5 reason = Timeout
78372024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 19 reason = Timeout
78382024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 55 reason = Timeout
78392024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
78402024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
78412024-11-21T19:42:17.836ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 12 reason = Timeout
78422024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
78432024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 55 reason = Timeout
78442024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
78452024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): activating background_task = instance_reincarnation iteration = 10 reason = Timeout
78462024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
78472024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 10 reason = Timeout
78482024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
78492024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 19 reason = Timeout
78502024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
78512024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 10 reason = Timeout
78522024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
78532024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 19 reason = Timeout
78542024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
78552024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 19 reason = Timeout
78562024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
78572024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
78582024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 19 reason = Timeout
78592024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 19 reason = Timeout
78602024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
78612024-11-21T19:42:17.837ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
78622024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
78632024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 19 reason = Timeout
78642024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
78652024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
78662024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 19 reason = Timeout
78672024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
78682024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
78692024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
78702024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
78712024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
78722024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
78732024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
78742024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
78752024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
78762024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
78772024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
78782024-11-21T19:42:17.837ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("4e669ff4-6dde-4cd2-8491-623460b56d8d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 1280, tv_nsec: 709849238 } })
78792024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
78802024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
78812024-11-21T19:42:17.837ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0377cf96-baf3-4e09-a570-eb7c67037d4d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0d1b3b31-4591-4bcb-931d-e8e03365f0d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:1::1))) }], valid_until: Instant { tv_sec: 1280, tv_nsec: 709911589 } })
78822024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
78832024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
78842024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
78852024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
78862024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
78872024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
78882024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
78892024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
78902024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
78912024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
78922024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
78932024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
78942024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
78952024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
78962024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
78972024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
78982024-11-21T19:42:17.837ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
78992024-11-21T19:42:17.838ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
79002024-11-21T19:42:17.838ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 1.394988ms iteration = 10 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
79012024-11-21T19:42:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
79022024-11-21T19:42:17.838ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
79032024-11-21T19:42:17.838ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } dns_group = internal
79042024-11-21T19:42:17.838ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 1.340258ms iteration = 10 status = {"addresses":["[fd00:1122:3344:3::1]:5353","[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353"]}
79052024-11-21T19:42:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
79062024-11-21T19:42:17.840ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1280, tv_nsec: 712508543 } })
79072024-11-21T19:42:17.840ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
79082024-11-21T19:42:17.840ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
79092024-11-21T19:42:17.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
79102024-11-21T19:42:17.840ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
79112024-11-21T19:42:17.841ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 4.50837ms iteration = 19 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
79122024-11-21T19:42:17.841ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
79132024-11-21T19:42:17.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
79142024-11-21T19:42:17.842ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
79152024-11-21T19:42:17.842ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1280, tv_nsec: 714901295 } })
79162024-11-21T19:42:17.842ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
79172024-11-21T19:42:17.843ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
79182024-11-21T19:42:17.843ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
79192024-11-21T19:42:17.843ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
79202024-11-21T19:42:17.845ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
79212024-11-21T19:42:17.845ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
79222024-11-21T19:42:17.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
79232024-11-21T19:42:17.846ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-11-21 19:32:17.836688804 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
79242024-11-21T19:42:17.846ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 9.444046ms iteration = 10 status = {"expiration":"2024-11-21T19:32:17.836688804Z","pruned":[]}
79252024-11-21T19:42:17.846ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
79262024-11-21T19:42:17.848ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
79272024-11-21T19:42:17.848ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
79282024-11-21T19:42:17.848ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 11.456794ms iteration = 19 status = {"errors":[],"garbage_collect_requested":[]}
79292024-11-21T19:42:17.848ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
79302024-11-21T19:42:17.848ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
79312024-11-21T19:42:17.849ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
79322024-11-21T19:42:17.851ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
79332024-11-21T19:42:17.851ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
79342024-11-21T19:42:17.851ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
79352024-11-21T19:42:17.852ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
79362024-11-21T19:42:17.852ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
79372024-11-21T19:42:17.852ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
79382024-11-21T19:42:17.856ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 19.071935ms iteration = 19 status = {"errors":[],"records_set_to_done":[]}
79392024-11-21T19:42:17.856ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 19.354818ms iteration = 10 status = {"errors":[],"found_port_ok":[]}
79402024-11-21T19:42:17.859ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
79412024-11-21T19:42:17.862ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
79422024-11-21T19:42:17.862ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
79432024-11-21T19:42:17.863ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
79442024-11-21T19:42:17.884ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
79452024-11-21T19:42:17.885ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
79462024-11-21T19:42:17.887ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
79472024-11-21T19:42:17.888ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
79482024-11-21T19:42:17.888ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
79492024-11-21T19:42:17.891ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
79502024-11-21T19:42:17.891ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
79512024-11-21T19:42:17.891ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
79522024-11-21T19:42:17.894ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
79532024-11-21T19:42:17.895ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
79542024-11-21T19:42:17.900ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
79552024-11-21T19:42:17.900ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
79562024-11-21T19:42:17.901ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
79572024-11-21T19:42:17.903ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0 total = 0
79582024-11-21T19:42:17.903ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:171 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0 vmms_found = 0
79592024-11-21T19:42:17.903ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 66.137844ms iteration = 10 status = {"errors":[],"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0,"vmms_found":0}
79602024-11-21T19:42:17.904ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
79612024-11-21T19:42:17.904ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
79622024-11-21T19:42:17.904ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
79632024-11-21T19:42:17.905ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
79642024-11-21T19:42:17.906ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
79652024-11-21T19:42:17.906ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
79662024-11-21T19:42:17.907ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
79672024-11-21T19:42:17.908ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
79682024-11-21T19:42:17.908ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
79692024-11-21T19:42:17.909ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
79702024-11-21T19:42:17.909ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
79712024-11-21T19:42:17.909ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
79722024-11-21T19:42:17.910ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
79732024-11-21T19:42:17.911ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
79742024-11-21T19:42:17.911ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
79752024-11-21T19:42:17.912ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
79762024-11-21T19:42:17.912ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
79772024-11-21T19:42:17.912ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
79782024-11-21T19:42:17.913ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
79792024-11-21T19:42:17.913ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
79802024-11-21T19:42:17.913ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
79812024-11-21T19:42:17.913ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
79822024-11-21T19:42:17.913ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
79832024-11-21T19:42:17.913ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
79842024-11-21T19:42:17.915ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
79852024-11-21T19:42:17.915ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
79862024-11-21T19:42:17.916ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
79872024-11-21T19:42:17.916ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
79882024-11-21T19:42:17.916ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
79892024-11-21T19:42:17.916ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
79902024-11-21T19:42:17.916ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
79912024-11-21T19:42:17.916ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
79922024-11-21T19:42:17.917ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
79932024-11-21T19:42:17.917ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
79942024-11-21T19:42:17.917ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
79952024-11-21T19:42:17.917ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
79962024-11-21T19:42:17.917ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
79972024-11-21T19:42:17.917ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
79982024-11-21T19:42:17.917ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
79992024-11-21T19:42:17.917ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
80002024-11-21T19:42:17.917ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
80012024-11-21T19:42:17.918ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
80022024-11-21T19:42:17.918ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
80032024-11-21T19:42:17.918ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
80042024-11-21T19:42:17.918ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
80052024-11-21T19:42:17.918ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 82.081736ms iteration = 10 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
80062024-11-21T19:42:17.920ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
80072024-11-21T19:42:17.920ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
80082024-11-21T19:42:17.923ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
80092024-11-21T19:42:17.923ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
80102024-11-21T19:42:17.923ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
80112024-11-21T19:42:17.924ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
80122024-11-21T19:42:17.925ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
80132024-11-21T19:42:17.927ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Fleet result = Ok(())
80142024-11-21T19:42:17.927ZDEBGnexus (BackgroundTasks): roles background_task = crdb_node_id_collector roles = RoleSet { roles: {} }
80152024-11-21T19:42:17.927ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = crdb_node_id_collector resource = Database result = Ok(())
80162024-11-21T19:42:17.930ZDEBGnexus (BackgroundTasks): activation complete background_task = crdb_node_id_collector elapsed = 93.934915ms iteration = 5 status = {"nsuccess":5}
80172024-11-21T19:42:17.931ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1280, tv_nsec: 804019206 } })
80182024-11-21T19:42:17.931ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
80192024-11-21T19:42:17.937ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
80202024-11-21T19:42:17.937ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
80212024-11-21T19:42:17.950ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
80222024-11-21T19:42:17.951ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
80232024-11-21T19:42:17.953ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
80242024-11-21T19:42:17.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
80252024-11-21T19:42:17.955ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
80262024-11-21T19:42:17.957ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
80272024-11-21T19:42:17.958ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 122.261383ms iteration = 19 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
80282024-11-21T19:42:17.959ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
80292024-11-21T19:42:17.959ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
80302024-11-21T19:42:17.960ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
80312024-11-21T19:42:17.960ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
80322024-11-21T19:42:17.960ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
80332024-11-21T19:42:17.960ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
80342024-11-21T19:42:17.962ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 125.724339ms iteration = 19 status = 0
80352024-11-21T19:42:17.964ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 127.081318ms iteration = 19 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
80362024-11-21T19:42:17.967ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
80372024-11-21T19:42:17.968ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
80382024-11-21T19:42:17.968ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
80392024-11-21T19:42:17.968ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
80402024-11-21T19:42:17.968ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
80412024-11-21T19:42:17.968ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
80422024-11-21T19:42:17.971ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
80432024-11-21T19:42:17.972ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
80442024-11-21T19:42:17.972ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
80452024-11-21T19:42:17.972ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
80462024-11-21T19:42:17.972ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
80472024-11-21T19:42:17.972ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
80482024-11-21T19:42:17.973ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
80492024-11-21T19:42:17.973ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 136.604725ms iteration = 55 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
80502024-11-21T19:42:17.973ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 136.592125ms iteration = 19 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
80512024-11-21T19:42:17.974ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
80522024-11-21T19:42:17.974ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
80532024-11-21T19:42:17.974ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
80542024-11-21T19:42:17.974ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
80552024-11-21T19:42:17.974ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
80562024-11-21T19:42:17.975ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
80572024-11-21T19:42:17.976ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
80582024-11-21T19:42:17.977ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
80592024-11-21T19:42:17.977ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
80602024-11-21T19:42:17.977ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
80612024-11-21T19:42:17.979ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
80622024-11-21T19:42:17.979ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 142.327812ms iteration = 55 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
80632024-11-21T19:42:17.979ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
80642024-11-21T19:42:17.981ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
80652024-11-21T19:42:17.981ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
80662024-11-21T19:42:17.981ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
80672024-11-21T19:42:17.982ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
80682024-11-21T19:42:17.982ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 145.15514ms iteration = 19 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
80692024-11-21T19:42:17.983ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
80702024-11-21T19:42:17.983ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 0 version = 2
80712024-11-21T19:42:17.983ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
80722024-11-21T19:42:17.984ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
80732024-11-21T19:42:17.984ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
80742024-11-21T19:42:17.984ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
80752024-11-21T19:42:17.984ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
80762024-11-21T19:42:17.986ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
80772024-11-21T19:42:17.989ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
80782024-11-21T19:42:17.989ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
80792024-11-21T19:42:17.989ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
80802024-11-21T19:42:17.990ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
80812024-11-21T19:42:17.990ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
80822024-11-21T19:42:17.991ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "99ac1134-25c2-41e3-a2ad-b720a9f5670a", "content-length": "26", "date": "Thu, 21 Nov 2024 19:42:17 GMT"} })
80832024-11-21T19:42:17.991ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
80842024-11-21T19:42:17.991ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 1 version = 2
80852024-11-21T19:42:17.991ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
80862024-11-21T19:42:17.991ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-11-21 19:33:32.758954 UTC dns_group = external generation = 2
80872024-11-21T19:42:17.991ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 154.592355ms iteration = 10 status = {"generation":2}
80882024-11-21T19:42:17.993ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
80892024-11-21T19:42:18.002ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
80902024-11-21T19:42:18.011ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
80912024-11-21T19:42:18.011ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
80922024-11-21T19:42:18.011ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 174.712304ms iteration = 19 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
80932024-11-21T19:42:18.020ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
80942024-11-21T19:42:18.021ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
80952024-11-21T19:42:18.021ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
80962024-11-21T19:42:18.021ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
80972024-11-21T19:42:18.032ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-11-21T19:33:15.446964Z, time_modified: 2024-11-21T19:33:15.446964Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }) }
80982024-11-21T19:42:18.032ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 195.866797ms iteration = 10 status = {"by_dns_name":{"default-silo.sys.oxide.test":{"silo_id":"001de000-5110-4000-8000-000000000000","tls_certs":[]},"recovery.sys.oxide.test":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]}},"default_endpoint":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]},"warnings":["silo 001de000-5110-4000-8000-000000000000 with DNS name \\"default-silo.sys.oxide.test\\" has no usable certificates"]}
80992024-11-21T19:42:18.035ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
81002024-11-21T19:42:18.045ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
81012024-11-21T19:42:18.045ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
81022024-11-21T19:42:18.049ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
81032024-11-21T19:42:18.051ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
81042024-11-21T19:42:18.051ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
81052024-11-21T19:42:18.052ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
81062024-11-21T19:42:18.061ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
81072024-11-21T19:42:18.064ZWARNnexus (BackgroundTasks): instance reincarnation completed with errors background_task = instance_reincarnation file = nexus/src/app/background/tasks/instance_reincarnation.rs:103 instances_changed_state = 0 instances_found = 0 instances_reincarnated = 0 query_errors = 0 restart_errors = 0
81082024-11-21T19:42:18.064ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_reincarnation elapsed = 227.281217ms iteration = 10 status = {"changed_state":[],"disabled":false,"errors":[],"instances_found":{"Failed":0,"SagaUnwound":0},"instances_reincarnated":[],"restart_errors":[]}
81092024-11-21T19:42:18.065ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
81102024-11-21T19:42:18.065ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
81112024-11-21T19:42:18.065ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
81122024-11-21T19:42:18.065ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
81132024-11-21T19:42:18.065ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 229.295724ms iteration = 20 status = {"physical_disks_added":0}
81142024-11-21T19:42:18.066ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "3c81d4bd-86a2-4a1e-82dc-237c7742fbd7", "content-length": "26", "date": "Thu, 21 Nov 2024 19:42:17 GMT"} })
81152024-11-21T19:42:18.066ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
81162024-11-21T19:42:18.066ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
81172024-11-21T19:42:18.067ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
81182024-11-21T19:42:18.068ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 51 version = 1
81192024-11-21T19:42:18.068ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
81202024-11-21T19:42:18.068ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-11-21 19:33:31.797390 UTC dns_group = internal generation = 1
81212024-11-21T19:42:18.068ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 231.878168ms iteration = 10 status = {"generation":1}
81222024-11-21T19:42:18.084ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
81232024-11-21T19:42:18.086ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
81242024-11-21T19:42:18.086ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
81252024-11-21T19:42:18.086ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
81262024-11-21T19:42:18.229ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
81272024-11-21T19:42:18.229ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
81282024-11-21T19:42:18.230ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "03137102-af5a-4643-8ad6-8b207e8b3db5", "content-length": "26", "date": "Thu, 21 Nov 2024 19:42:17 GMT"} })
81292024-11-21T19:42:18.230ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
81302024-11-21T19:42:18.231ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
81312024-11-21T19:42:18.297ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
81322024-11-21T19:42:18.311ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 474.49942ms iteration = 19 status = {}
81332024-11-21T19:42:18.323ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
81342024-11-21T19:42:18.323ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
81352024-11-21T19:42:18.323ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "51dd8574-88de-4e4c-9853-ad0c58be5f53", "content-length": "372", "date": "Thu, 21 Nov 2024 19:42:18 GMT"} })
81362024-11-21T19:42:18.324ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
81372024-11-21T19:42:18.324ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
81382024-11-21T19:42:18.360ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
81392024-11-21T19:42:18.360ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::8]:5353/config", status: 204, headers: {"x-request-id": "54e4cf1c-71a1-4821-91f7-fbf1e96a22f4", "date": "Thu, 21 Nov 2024 19:42:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
81402024-11-21T19:42:18.360ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
81412024-11-21T19:42:18.362ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::9]:5353/config", status: 204, headers: {"x-request-id": "88fe1ba8-239c-4a96-aac4-912849c3d991", "date": "Thu, 21 Nov 2024 19:42:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
81422024-11-21T19:42:18.362ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
81432024-11-21T19:42:18.363ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
81442024-11-21T19:42:18.363ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 526.946431ms iteration = 12 status = {"generation":2,"server_results":{"[fd00:1122:3344:101::8]:5353":{"Ok":null},"[fd00:1122:3344:101::9]:5353":{"Ok":null}}}
81452024-11-21T19:42:18.365ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
81462024-11-21T19:42:18.365ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
81472024-11-21T19:42:18.373ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
81482024-11-21T19:42:18.373ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "ea90c945-c014-4e70-b176-d221bd8676fa", "content-length": "1", "date": "Thu, 21 Nov 2024 19:42:18 GMT"} })
81492024-11-21T19:42:18.375ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
81502024-11-21T19:42:18.376ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
81512024-11-21T19:42:18.378ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 541.426944ms iteration = 19 status = null
81522024-11-21T19:42:18.409ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
81532024-11-21T19:42:18.410ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:2::1]:5353/config", status: 204, headers: {"x-request-id": "60fd6611-17ca-40c6-b215-4d70974b9e27", "date": "Thu, 21 Nov 2024 19:42:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
81542024-11-21T19:42:18.410ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
81552024-11-21T19:42:18.412ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:1::1]:5353/config", status: 204, headers: {"x-request-id": "f4baf2f7-8820-4762-99e5-7b53502a02ed", "date": "Thu, 21 Nov 2024 19:42:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
81562024-11-21T19:42:18.412ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
81572024-11-21T19:42:18.413ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:3::1]:5353/config", status: 204, headers: {"x-request-id": "d1ed613e-5ff1-4f28-8a61-d331de81e48c", "date": "Thu, 21 Nov 2024 19:42:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
81582024-11-21T19:42:18.413ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
81592024-11-21T19:42:18.414ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
81602024-11-21T19:42:18.414ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 577.935953ms iteration = 12 status = {"generation":1,"server_results":{"[fd00:1122:3344:1::1]:5353":{"Ok":null},"[fd00:1122:3344:2::1]:5353":{"Ok":null},"[fd00:1122:3344:3::1]:5353":{"Ok":null}}}
81612024-11-21T19:42:18.422ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
81622024-11-21T19:42:18.426ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "75de3b66-0275-41b7-91cc-d26b118996e5", "date": "Thu, 21 Nov 2024 19:42:17 GMT"} })
81632024-11-21T19:42:18.426ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
81642024-11-21T19:42:18.426ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
81652024-11-21T19:42:18.426ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "8a508f01-795b-4c1d-86c8-5000e5498a2c", "content-length": "691", "date": "Thu, 21 Nov 2024 19:42:18 GMT"} })
81662024-11-21T19:42:18.426ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
81672024-11-21T19:42:18.426ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
81682024-11-21T19:42:18.426ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
81692024-11-21T19:42:18.428ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 591.342861ms iteration = 19 status = {}
81702024-11-21T19:42:18.428ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
81712024-11-21T19:42:18.428ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
81722024-11-21T19:42:18.432ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
81732024-11-21T19:42:18.432ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
81742024-11-21T19:42:18.435ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
81752024-11-21T19:42:18.435ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
81762024-11-21T19:42:18.436ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "71a3b7de-4705-4198-8446-bab91677f723", "date": "Thu, 21 Nov 2024 19:42:17 GMT"} })
81772024-11-21T19:42:18.437ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 600.770498ms iteration = 19 status = {}
81782024-11-21T19:42:18.481ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
81792024-11-21T19:42:18.482ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
81802024-11-21T19:42:18.483ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
81812024-11-21T19:42:18.483ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
81822024-11-21T19:42:18.483ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
81832024-11-21T19:42:18.494ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
81842024-11-21T19:42:18.494ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "fd4ae7b7-94c3-42d2-82b6-bb117da66e2e", "content-length": "134", "date": "Thu, 21 Nov 2024 19:42:18 GMT"} })
81852024-11-21T19:42:18.494ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
81862024-11-21T19:42:18.494ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
81872024-11-21T19:42:18.494ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
81882024-11-21T19:42:18.494ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
81892024-11-21T19:42:18.494ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
81902024-11-21T19:42:18.494ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
81912024-11-21T19:42:18.494ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
81922024-11-21T19:42:18.494ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
81932024-11-21T19:42:18.494ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
81942024-11-21T19:42:18.494ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
81952024-11-21T19:42:18.494ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
81962024-11-21T19:42:18.495ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
81972024-11-21T19:42:18.495ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
81982024-11-21T19:42:18.497ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
81992024-11-21T19:42:18.497ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "da625ba6-af01-4dd2-8a22-e02a32fe85d2", "content-length": "2", "date": "Thu, 21 Nov 2024 19:42:18 GMT"} })
82002024-11-21T19:42:18.497ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
82012024-11-21T19:42:18.497ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "78e50e70-f4bc-4407-8532-55fd055dc8c9", "content-length": "51", "date": "Thu, 21 Nov 2024 19:42:18 GMT"} })
82022024-11-21T19:42:18.497ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
82032024-11-21T19:42:18.497ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
82042024-11-21T19:42:18.596ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "43913cb6-fa3e-461a-aca0-2a4ce1f5c621", "date": "Thu, 21 Nov 2024 19:42:17 GMT"} })
82052024-11-21T19:42:18.596ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
82062024-11-21T19:42:18.596ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
82072024-11-21T19:42:18.596ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "385eedaf-fb29-4d2b-b469-54df73436e95", "content-length": "682", "date": "Thu, 21 Nov 2024 19:42:17 GMT"} })
82082024-11-21T19:42:18.596ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
82092024-11-21T19:42:18.597ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
82102024-11-21T19:42:18.600ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
82112024-11-21T19:42:18.600ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
82122024-11-21T19:42:18.604ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
82132024-11-21T19:42:18.604ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
82142024-11-21T19:42:18.606ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
82152024-11-21T19:42:18.606ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
82162024-11-21T19:42:18.613ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 777.014282ms iteration = 19 status = {}
82172024-11-21T19:42:26.965ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 394 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 7f82d124-d448-4101-b9e5-f3fa59709f6f response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
82182024-11-21T19:42:27.836ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 56 reason = Timeout
82192024-11-21T19:42:27.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
82202024-11-21T19:42:27.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
82212024-11-21T19:42:27.837ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 56 reason = Timeout
82222024-11-21T19:42:27.837ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
82232024-11-21T19:42:27.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
82242024-11-21T19:42:27.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
82252024-11-21T19:42:27.840ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
82262024-11-21T19:42:27.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
82272024-11-21T19:42:27.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
82282024-11-21T19:42:27.843ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
82292024-11-21T19:42:27.843ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
82302024-11-21T19:42:27.844ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
82312024-11-21T19:42:27.847ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
82322024-11-21T19:42:27.847ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
82332024-11-21T19:42:27.847ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
82342024-11-21T19:42:27.859ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
82352024-11-21T19:42:27.859ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 23.683276ms iteration = 56 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
82362024-11-21T19:42:27.866ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
82372024-11-21T19:42:27.866ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
82382024-11-21T19:42:27.868ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 31.542461ms iteration = 56 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
82392024-11-21T19:42:37.836ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 57 reason = Timeout
82402024-11-21T19:42:37.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
82412024-11-21T19:42:37.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
82422024-11-21T19:42:37.837ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 57 reason = Timeout
82432024-11-21T19:42:37.837ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
82442024-11-21T19:42:37.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
82452024-11-21T19:42:37.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
82462024-11-21T19:42:37.839ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
82472024-11-21T19:42:37.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
82482024-11-21T19:42:37.841ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
82492024-11-21T19:42:37.841ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
82502024-11-21T19:42:37.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
82512024-11-21T19:42:37.841ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 3.993623ms iteration = 57 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
82522024-11-21T19:42:37.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
82532024-11-21T19:42:37.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
82542024-11-21T19:42:37.843ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
82552024-11-21T19:42:37.846ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
82562024-11-21T19:42:37.846ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
82572024-11-21T19:42:37.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
82582024-11-21T19:42:37.858ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
82592024-11-21T19:42:37.858ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 21.946254ms iteration = 57 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
82602024-11-21T19:42:41.966ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 495 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = fc9d62e5-aea1-4e9f-87a0-1d44cf1e0fa5 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
82612024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 20 reason = Timeout
82622024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 58 reason = Timeout
82632024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
82642024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 21 reason = Timeout
82652024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
82662024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
82672024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 20 reason = Timeout
82682024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 20 reason = Timeout
82692024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
82702024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 20 reason = Timeout
82712024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
82722024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 20 reason = Timeout
82732024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
82742024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 20 reason = Timeout
82752024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
82762024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 20 reason = Timeout
82772024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
82782024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
82792024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
82802024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
82812024-11-21T19:42:47.836ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
82822024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
82832024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
82842024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
82852024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
82862024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 58 reason = Timeout
82872024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 20 reason = Timeout
82882024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
82892024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
82902024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 20 reason = Timeout
82912024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
82922024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 20 reason = Timeout
82932024-11-21T19:42:47.837ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
82942024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 20 reason = Timeout
82952024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
82962024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
82972024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 20 reason = Timeout
82982024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 20 reason = Timeout
82992024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
83002024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
83012024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 20 reason = Timeout
83022024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
83032024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
83042024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
83052024-11-21T19:42:47.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
83062024-11-21T19:42:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
83072024-11-21T19:42:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
83082024-11-21T19:42:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
83092024-11-21T19:42:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
83102024-11-21T19:42:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
83112024-11-21T19:42:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
83122024-11-21T19:42:47.840ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1310, tv_nsec: 712109558 } })
83132024-11-21T19:42:47.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
83142024-11-21T19:42:47.840ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
83152024-11-21T19:42:47.840ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 4.49698ms iteration = 20 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
83162024-11-21T19:42:47.841ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1310, tv_nsec: 713116681 } })
83172024-11-21T19:42:47.841ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
83182024-11-21T19:42:47.841ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
83192024-11-21T19:42:47.842ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
83202024-11-21T19:42:47.844ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
83212024-11-21T19:42:47.844ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
83222024-11-21T19:42:47.844ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
83232024-11-21T19:42:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
83242024-11-21T19:42:47.845ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
83252024-11-21T19:42:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
83262024-11-21T19:42:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
83272024-11-21T19:42:47.845ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
83282024-11-21T19:42:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
83292024-11-21T19:42:47.847ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 9.915362ms iteration = 20 status = {"errors":[],"garbage_collect_requested":[]}
83302024-11-21T19:42:47.848ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 10.821454ms iteration = 20 status = {"errors":[],"records_set_to_done":[]}
83312024-11-21T19:42:47.848ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
83322024-11-21T19:42:47.850ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
83332024-11-21T19:42:47.850ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
83342024-11-21T19:42:47.850ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
83352024-11-21T19:42:47.853ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
83362024-11-21T19:42:47.853ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
83372024-11-21T19:42:47.854ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
83382024-11-21T19:42:47.854ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
83392024-11-21T19:42:47.854ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
83402024-11-21T19:42:47.854ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
83412024-11-21T19:42:47.854ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
83422024-11-21T19:42:47.854ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
83432024-11-21T19:42:47.856ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1310, tv_nsec: 727985760 } })
83442024-11-21T19:42:47.856ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
83452024-11-21T19:42:47.857ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
83462024-11-21T19:42:47.859ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
83472024-11-21T19:42:47.859ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
83482024-11-21T19:42:47.859ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
83492024-11-21T19:42:47.884ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
83502024-11-21T19:42:47.887ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
83512024-11-21T19:42:47.887ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
83522024-11-21T19:42:47.887ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
83532024-11-21T19:42:47.889ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
83542024-11-21T19:42:47.891ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
83552024-11-21T19:42:47.891ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
83562024-11-21T19:42:47.891ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
83572024-11-21T19:42:47.891ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
83582024-11-21T19:42:47.892ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
83592024-11-21T19:42:47.892ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
83602024-11-21T19:42:47.893ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
83612024-11-21T19:42:47.896ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
83622024-11-21T19:42:47.896ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
83632024-11-21T19:42:47.896ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
83642024-11-21T19:42:47.897ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
83652024-11-21T19:42:47.902ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
83662024-11-21T19:42:47.903ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
83672024-11-21T19:42:47.903ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
83682024-11-21T19:42:47.903ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
83692024-11-21T19:42:47.904ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
83702024-11-21T19:42:47.905ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
83712024-11-21T19:42:47.905ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
83722024-11-21T19:42:47.906ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 68.551816ms iteration = 58 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
83732024-11-21T19:42:47.907ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
83742024-11-21T19:42:47.910ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
83752024-11-21T19:42:47.910ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
83762024-11-21T19:42:47.910ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
83772024-11-21T19:42:47.911ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
83782024-11-21T19:42:47.911ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
83792024-11-21T19:42:47.911ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
83802024-11-21T19:42:47.911ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
83812024-11-21T19:42:47.911ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
83822024-11-21T19:42:47.917ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
83832024-11-21T19:42:47.917ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 79.740995ms iteration = 20 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
83842024-11-21T19:42:47.917ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
83852024-11-21T19:42:47.917ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
83862024-11-21T19:42:47.918ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "f7258238-e12a-4b3a-b6b2-94e09bdb03f2", "content-length": "26", "date": "Thu, 21 Nov 2024 19:42:47 GMT"} })
83872024-11-21T19:42:47.918ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
83882024-11-21T19:42:47.919ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
83892024-11-21T19:42:47.929ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
83902024-11-21T19:42:47.929ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 92.907341ms iteration = 58 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
83912024-11-21T19:42:47.940ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
83922024-11-21T19:42:47.940ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
83932024-11-21T19:42:47.941ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 103.839407ms iteration = 20 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
83942024-11-21T19:42:47.943ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
83952024-11-21T19:42:47.943ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
83962024-11-21T19:42:47.944ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 107.649979ms iteration = 20 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
83972024-11-21T19:42:47.944ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
83982024-11-21T19:42:47.944ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
83992024-11-21T19:42:47.945ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "d3947cc7-d0f3-45c6-8a0b-567dfc8e43d6", "content-length": "26", "date": "Thu, 21 Nov 2024 19:42:47 GMT"} })
84002024-11-21T19:42:47.945ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
84012024-11-21T19:42:47.945ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 107.75236ms iteration = 20 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
84022024-11-21T19:42:47.947ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
84032024-11-21T19:42:47.947ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
84042024-11-21T19:42:47.947ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
84052024-11-21T19:42:47.948ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "6ca4b686-d2c7-4945-a6cc-e7dc4067d256", "content-length": "26", "date": "Thu, 21 Nov 2024 19:42:47 GMT"} })
84062024-11-21T19:42:47.948ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
84072024-11-21T19:42:47.950ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
84082024-11-21T19:42:47.951ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
84092024-11-21T19:42:47.951ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
84102024-11-21T19:42:47.951ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 114.004853ms iteration = 20 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
84112024-11-21T19:42:47.976ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 140.027101ms iteration = 21 status = {"physical_disks_added":0}
84122024-11-21T19:42:47.985ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
84132024-11-21T19:42:47.988ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
84142024-11-21T19:42:47.988ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
84152024-11-21T19:42:47.989ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
84162024-11-21T19:42:47.991ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 155.338386ms iteration = 20 status = 0
84172024-11-21T19:42:48.000ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
84182024-11-21T19:42:48.000ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
84192024-11-21T19:42:48.010ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
84202024-11-21T19:42:48.011ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
84212024-11-21T19:42:48.019ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
84222024-11-21T19:42:48.019ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "511359ca-13c6-4d9c-95c2-3713ed87ca86", "content-length": "1", "date": "Thu, 21 Nov 2024 19:42:47 GMT"} })
84232024-11-21T19:42:48.021ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
84242024-11-21T19:42:48.021ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
84252024-11-21T19:42:48.098ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
84262024-11-21T19:42:48.098ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
84272024-11-21T19:42:48.098ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "01426a4f-131c-478f-a856-78b0f47cbff8", "content-length": "372", "date": "Thu, 21 Nov 2024 19:42:48 GMT"} })
84282024-11-21T19:42:48.098ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
84292024-11-21T19:42:48.099ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
84302024-11-21T19:42:48.099ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
84312024-11-21T19:42:48.100ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 264.402893ms iteration = 20 status = null
84322024-11-21T19:42:48.101ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
84332024-11-21T19:42:48.101ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
84342024-11-21T19:42:48.102ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
84352024-11-21T19:42:48.103ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
84362024-11-21T19:42:48.105ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
84372024-11-21T19:42:48.105ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
84382024-11-21T19:42:48.105ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
84392024-11-21T19:42:48.115ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 279.514284ms iteration = 20 status = {}
84402024-11-21T19:42:48.161ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
84412024-11-21T19:42:48.161ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
84422024-11-21T19:42:48.161ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "9fa78371-4382-41fc-8c1c-202e7e3c7a47", "date": "Thu, 21 Nov 2024 19:42:48 GMT"} })
84432024-11-21T19:42:48.161ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
84442024-11-21T19:42:48.162ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
84452024-11-21T19:42:48.164ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
84462024-11-21T19:42:48.164ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
84472024-11-21T19:42:48.168ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
84482024-11-21T19:42:48.168ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
84492024-11-21T19:42:48.171ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
84502024-11-21T19:42:48.171ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
84512024-11-21T19:42:48.171ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "7749a250-a182-4311-891f-976a53ef1dd2", "date": "Thu, 21 Nov 2024 19:42:48 GMT"} })
84522024-11-21T19:42:48.173ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 335.426721ms iteration = 20 status = {}
84532024-11-21T19:42:48.206ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
84542024-11-21T19:42:48.207ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
84552024-11-21T19:42:48.209ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
84562024-11-21T19:42:48.209ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
84572024-11-21T19:42:48.209ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
84582024-11-21T19:42:48.219ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
84592024-11-21T19:42:48.219ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "e239ed1a-b34d-4e8b-b158-14a58a19403c", "content-length": "691", "date": "Thu, 21 Nov 2024 19:42:48 GMT"} })
84602024-11-21T19:42:48.219ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
84612024-11-21T19:42:48.219ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
84622024-11-21T19:42:48.219ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
84632024-11-21T19:42:48.220ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 384.066132ms iteration = 20 status = {}
84642024-11-21T19:42:48.228ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
84652024-11-21T19:42:48.228ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "78fbe04d-b5c0-4147-8eef-98f1c73a2da1", "content-length": "134", "date": "Thu, 21 Nov 2024 19:42:47 GMT"} })
84662024-11-21T19:42:48.228ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
84672024-11-21T19:42:48.228ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
84682024-11-21T19:42:48.228ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
84692024-11-21T19:42:48.228ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
84702024-11-21T19:42:48.228ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
84712024-11-21T19:42:48.228ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
84722024-11-21T19:42:48.228ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
84732024-11-21T19:42:48.228ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
84742024-11-21T19:42:48.229ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
84752024-11-21T19:42:48.229ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
84762024-11-21T19:42:48.229ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
84772024-11-21T19:42:48.229ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
84782024-11-21T19:42:48.229ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
84792024-11-21T19:42:48.230ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
84802024-11-21T19:42:48.231ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "0d5ccb95-1510-4fd1-af46-509ce9eb46bd", "content-length": "2", "date": "Thu, 21 Nov 2024 19:42:47 GMT"} })
84812024-11-21T19:42:48.231ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
84822024-11-21T19:42:48.231ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "8bda323b-cf46-44e0-b9f2-18963cd8237c", "content-length": "51", "date": "Thu, 21 Nov 2024 19:42:47 GMT"} })
84832024-11-21T19:42:48.231ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
84842024-11-21T19:42:48.231ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
84852024-11-21T19:42:48.328ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "7cb21b26-bec5-4d16-a69e-5cb61f34400b", "date": "Thu, 21 Nov 2024 19:42:48 GMT"} })
84862024-11-21T19:42:48.328ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
84872024-11-21T19:42:48.328ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
84882024-11-21T19:42:48.328ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "0da11059-bcee-47df-a38b-1d4617ae6ab3", "content-length": "682", "date": "Thu, 21 Nov 2024 19:42:48 GMT"} })
84892024-11-21T19:42:48.328ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
84902024-11-21T19:42:48.329ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
84912024-11-21T19:42:48.332ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
84922024-11-21T19:42:48.332ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
84932024-11-21T19:42:48.335ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
84942024-11-21T19:42:48.335ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
84952024-11-21T19:42:48.337ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
84962024-11-21T19:42:48.338ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
84972024-11-21T19:42:48.344ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 508.315392ms iteration = 20 status = {}
84982024-11-21T19:42:56.964ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 483 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 0ffd6f62-a2bb-4045-bc8e-7f5a43a2532f response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
84992024-11-21T19:42:57.836ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 59 reason = Timeout
85002024-11-21T19:42:57.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
85012024-11-21T19:42:57.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
85022024-11-21T19:42:57.837ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 59 reason = Timeout
85032024-11-21T19:42:57.837ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
85042024-11-21T19:42:57.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
85052024-11-21T19:42:57.839ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
85062024-11-21T19:42:57.839ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
85072024-11-21T19:42:57.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
85082024-11-21T19:42:57.841ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
85092024-11-21T19:42:57.841ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
85102024-11-21T19:42:57.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
85112024-11-21T19:42:57.841ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 4.092055ms iteration = 59 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
85122024-11-21T19:42:57.843ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
85132024-11-21T19:42:57.844ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
85142024-11-21T19:42:57.845ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
85152024-11-21T19:42:57.847ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
85162024-11-21T19:42:57.847ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
85172024-11-21T19:42:57.848ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
85182024-11-21T19:42:57.857ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
85192024-11-21T19:42:57.857ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 20.348152ms iteration = 59 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
85202024-11-21T19:43:07.836ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 60 reason = Timeout
85212024-11-21T19:43:07.836ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
85222024-11-21T19:43:07.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
85232024-11-21T19:43:07.837ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 60 reason = Timeout
85242024-11-21T19:43:07.837ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
85252024-11-21T19:43:07.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
85262024-11-21T19:43:07.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
85272024-11-21T19:43:07.840ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
85282024-11-21T19:43:07.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
85292024-11-21T19:43:07.841ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
85302024-11-21T19:43:07.841ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
85312024-11-21T19:43:07.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
85322024-11-21T19:43:07.842ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 4.337488ms iteration = 60 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
85332024-11-21T19:43:07.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
85342024-11-21T19:43:07.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
85352024-11-21T19:43:07.844ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
85362024-11-21T19:43:07.847ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
85372024-11-21T19:43:07.847ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
85382024-11-21T19:43:07.847ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
85392024-11-21T19:43:07.858ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
85402024-11-21T19:43:07.858ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 22.392229ms iteration = 60 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
85412024-11-21T19:43:11.965ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 278 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 3e4de93e-e9f5-45d6-bea5-89d4d38f6a54 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
85422024-11-21T19:43:17.836ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 21 reason = Timeout
85432024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
85442024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 11 reason = Timeout
85452024-11-21T19:43:17.837ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
85462024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 12 reason = Timeout
85472024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
85482024-11-21T19:43:17.837ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:83 reason = blueprint disabled target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb
85492024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 18.75\u{b5}s iteration = 12 status = {"enabled":false,"target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb"}
85502024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 11 reason = Timeout
85512024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 21 reason = Timeout
85522024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
85532024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 21 reason = Timeout
85542024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
85552024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
85562024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
85572024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = inventory_collection iteration = 2 reason = Timeout
85582024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {} }
85592024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 11 reason = Timeout
85602024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 21 reason = Timeout
85612024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 13 reason = Timeout
85622024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 22 reason = Timeout
85632024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 11 reason = Timeout
85642024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 21 reason = Timeout
85652024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
85662024-11-21T19:43:17.837ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-11-21 19:33:17.837137467 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
85672024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
85682024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
85692024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 21 reason = Timeout
85702024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
85712024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 13 reason = Timeout
85722024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 11 reason = Timeout
85732024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
85742024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
85752024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 21 reason = Timeout
85762024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 11 reason = Timeout
85772024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
85782024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
85792024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 61 reason = Timeout
85802024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
85812024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 11 reason = Timeout
85822024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
85832024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Database result = Ok(())
85842024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
85852024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
85862024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
85872024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
85882024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 11 reason = Timeout
85892024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 61 reason = Timeout
85902024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
85912024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
85922024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 21 reason = Timeout
85932024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
85942024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
85952024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = instance_reincarnation iteration = 11 reason = Timeout
85962024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 21 reason = Timeout
85972024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
85982024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
85992024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 21 reason = Timeout
86002024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 11 reason = Timeout
86012024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
86022024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
86032024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
86042024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 21 reason = Timeout
86052024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
86062024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
86072024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
86082024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 21 reason = Timeout
86092024-11-21T19:43:17.837ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
86102024-11-21T19:43:17.837ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("4e669ff4-6dde-4cd2-8491-623460b56d8d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 1340, tv_nsec: 709240980 } })
86112024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): activating background_task = service_firewall_rule_propagation iteration = 3 reason = Timeout
86122024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
86132024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 21 reason = Timeout
86142024-11-21T19:43:17.838ZDEBGnexus (service-firewall-rule-progator): starting background task for service firewall rule propagation background_task = service_firewall_rule_propagation
86152024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
86162024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
86172024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
86182024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): activating background_task = saga_recovery iteration = 2 reason = Timeout
86192024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
86202024-11-21T19:43:17.838ZDEBGnexus (SagaRecoverer): listing candidate sagas for recovery
86212024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
86222024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 21 reason = Timeout
86232024-11-21T19:43:17.838ZDEBGnexus (SagaRecoverer): roles roles = RoleSet { roles: {} }
86242024-11-21T19:43:17.838ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
86252024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
86262024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
86272024-11-21T19:43:17.838ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0377cf96-baf3-4e09-a570-eb7c67037d4d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0d1b3b31-4591-4bcb-931d-e8e03365f0d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:1::1))) }], valid_until: Instant { tv_sec: 1340, tv_nsec: 709434172 } })
86282024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
86292024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
86302024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
86312024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
86322024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
86332024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
86342024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
86352024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
86362024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
86372024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
86382024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 1.213066ms iteration = 11 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
86392024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
86402024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
86412024-11-21T19:43:17.838ZDEBGnexus (SagaRecoverer): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000003a8a, .. }) resource = Database result = Ok(())
86422024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
86432024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } dns_group = internal
86442024-11-21T19:43:17.838ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 1.244977ms iteration = 11 status = {"addresses":["[fd00:1122:3344:3::1]:5353","[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353"]}
86452024-11-21T19:43:17.840ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1340, tv_nsec: 712164359 } })
86462024-11-21T19:43:17.840ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
86472024-11-21T19:43:17.840ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
86482024-11-21T19:43:17.841ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
86492024-11-21T19:43:17.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
86502024-11-21T19:43:17.841ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
86512024-11-21T19:43:17.841ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 4.808734ms iteration = 21 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
86522024-11-21T19:43:17.843ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1340, tv_nsec: 714774474 } })
86532024-11-21T19:43:17.843ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
86542024-11-21T19:43:17.844ZDEBGnexus (BackgroundTasks): authorize result action = Modify actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Inventory result = Ok(())
86552024-11-21T19:43:17.844ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {} }
86562024-11-21T19:43:17.844ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Database result = Ok(())
86572024-11-21T19:43:17.845ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
86582024-11-21T19:43:17.845ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
86592024-11-21T19:43:17.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
86602024-11-21T19:43:17.848ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
86612024-11-21T19:43:17.848ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
86622024-11-21T19:43:17.848ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
86632024-11-21T19:43:17.848ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-11-21 19:33:17.837137467 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
86642024-11-21T19:43:17.848ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 11.660865ms iteration = 11 status = {"expiration":"2024-11-21T19:33:17.837137467Z","pruned":[]}
86652024-11-21T19:43:17.848ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
86662024-11-21T19:43:17.849ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
86672024-11-21T19:43:17.850ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
86682024-11-21T19:43:17.852ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
86692024-11-21T19:43:17.852ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
86702024-11-21T19:43:17.852ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
86712024-11-21T19:43:17.852ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
86722024-11-21T19:43:17.852ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
86732024-11-21T19:43:17.852ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
86742024-11-21T19:43:17.853ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
86752024-11-21T19:43:17.853ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
86762024-11-21T19:43:17.853ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
86772024-11-21T19:43:17.854ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
86782024-11-21T19:43:17.854ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
86792024-11-21T19:43:17.854ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
86802024-11-21T19:43:17.854ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
86812024-11-21T19:43:17.854ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 17.268641ms iteration = 11 status = {"errors":[],"found_port_ok":[]}
86822024-11-21T19:43:17.855ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
86832024-11-21T19:43:17.856ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
86842024-11-21T19:43:17.856ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
86852024-11-21T19:43:17.858ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 20.379003ms iteration = 21 status = {"errors":[],"garbage_collect_requested":[]}
86862024-11-21T19:43:17.858ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
86872024-11-21T19:43:17.858ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0 total = 0
86882024-11-21T19:43:17.858ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:171 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0 vmms_found = 0
86892024-11-21T19:43:17.858ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 21.054051ms iteration = 11 status = {"errors":[],"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0,"vmms_found":0}
86902024-11-21T19:43:17.860ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
86912024-11-21T19:43:17.861ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
86922024-11-21T19:43:17.861ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
86932024-11-21T19:43:17.861ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
86942024-11-21T19:43:17.861ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 23.718357ms iteration = 21 status = {"errors":[],"records_set_to_done":[]}
86952024-11-21T19:43:17.861ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
86962024-11-21T19:43:17.862ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
86972024-11-21T19:43:17.862ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
86982024-11-21T19:43:17.862ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
86992024-11-21T19:43:17.862ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
87002024-11-21T19:43:17.863ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
87012024-11-21T19:43:17.863ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
87022024-11-21T19:43:17.863ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
87032024-11-21T19:43:17.864ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
87042024-11-21T19:43:17.865ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
87052024-11-21T19:43:17.868ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
87062024-11-21T19:43:17.869ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
87072024-11-21T19:43:17.869ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
87082024-11-21T19:43:17.870ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
87092024-11-21T19:43:17.870ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
87102024-11-21T19:43:17.870ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
87112024-11-21T19:43:17.870ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
87122024-11-21T19:43:17.871ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
87132024-11-21T19:43:17.872ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
87142024-11-21T19:43:17.872ZDEBGnexus (BackgroundTasks): inventory_prune_one: nothing eligible for removal (too few) background_task = inventory_collection candidates = [(0322ca29-70af-440b-84f5-66321d76596e, 0), (6eaed575-ff61-4803-aa79-b37ed649249f, 0), (e8597ba8-af28-46ea-9683-97e262ba7530, 0)]
87152024-11-21T19:43:17.872ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
87162024-11-21T19:43:17.872ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1340, tv_nsec: 744203307 } })
87172024-11-21T19:43:17.872ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _mgs._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_mgs._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_mgs._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12225, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1340, tv_nsec: 744336599 } })
87182024-11-21T19:43:17.872ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
87192024-11-21T19:43:17.872ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
87202024-11-21T19:43:17.873ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
87212024-11-21T19:43:17.873ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
87222024-11-21T19:43:17.873ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
87232024-11-21T19:43:17.874ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
87242024-11-21T19:43:17.874ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
87252024-11-21T19:43:17.876ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
87262024-11-21T19:43:17.876ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
87272024-11-21T19:43:17.876ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
87282024-11-21T19:43:17.876ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
87292024-11-21T19:43:17.876ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
87302024-11-21T19:43:17.876ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 38.496484ms iteration = 21 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
87312024-11-21T19:43:17.876ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
87322024-11-21T19:43:17.876ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
87332024-11-21T19:43:17.876ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
87342024-11-21T19:43:17.877ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
87352024-11-21T19:43:17.878ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
87362024-11-21T19:43:17.879ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
87372024-11-21T19:43:17.880ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
87382024-11-21T19:43:17.880ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
87392024-11-21T19:43:17.880ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
87402024-11-21T19:43:17.882ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
87412024-11-21T19:43:17.882ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
87422024-11-21T19:43:17.882ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 44.641666ms iteration = 21 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
87432024-11-21T19:43:17.882ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
87442024-11-21T19:43:17.883ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
87452024-11-21T19:43:17.883ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
87462024-11-21T19:43:17.883ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
87472024-11-21T19:43:17.885ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
87482024-11-21T19:43:17.885ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
87492024-11-21T19:43:17.885ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
87502024-11-21T19:43:17.886ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
87512024-11-21T19:43:17.887ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
87522024-11-21T19:43:17.887ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
87532024-11-21T19:43:17.888ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
87542024-11-21T19:43:17.888ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
87552024-11-21T19:43:17.891ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
87562024-11-21T19:43:17.891ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
87572024-11-21T19:43:17.891ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
87582024-11-21T19:43:17.891ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
87592024-11-21T19:43:17.891ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
87602024-11-21T19:43:17.891ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
87612024-11-21T19:43:17.893ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
87622024-11-21T19:43:17.893ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
87632024-11-21T19:43:17.895ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
87642024-11-21T19:43:17.895ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
87652024-11-21T19:43:17.895ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
87662024-11-21T19:43:17.895ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 0 version = 2
87672024-11-21T19:43:17.895ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
87682024-11-21T19:43:17.896ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
87692024-11-21T19:43:17.896ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
87702024-11-21T19:43:17.896ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
87712024-11-21T19:43:17.897ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
87722024-11-21T19:43:17.898ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
87732024-11-21T19:43:17.898ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
87742024-11-21T19:43:17.902ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
87752024-11-21T19:43:17.902ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
87762024-11-21T19:43:17.902ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
87772024-11-21T19:43:17.907ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
87782024-11-21T19:43:17.909ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 1 version = 2
87792024-11-21T19:43:17.909ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
87802024-11-21T19:43:17.909ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-11-21 19:33:32.758954 UTC dns_group = external generation = 2
87812024-11-21T19:43:17.909ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 72.093864ms iteration = 11 status = {"generation":2}
87822024-11-21T19:43:17.909ZINFOnexus (SagaRecoverer): listed in-progress sagas count = 0 file = nexus/src/app/background/tasks/saga_recovery.rs:438
87832024-11-21T19:43:17.909ZDEBGnexus (BackgroundTasks): activation complete background_task = saga_recovery elapsed = 72.060563ms iteration = 2 status = {"last_pass":{"Success":{"nfailed":0,"nfound":0,"nrecovered":0,"nremoved":0,"nskipped":0}},"ntotal_failures":0,"ntotal_finished":0,"ntotal_recovered":0,"ntotal_sec_errors_bad_state":0,"ntotal_sec_errors_missing":0,"ntotal_started":0,"recent_failures":[],"recent_recoveries":[]}
87842024-11-21T19:43:17.911ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
87852024-11-21T19:43:17.911ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
87862024-11-21T19:43:17.912ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
87872024-11-21T19:43:17.912ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
87882024-11-21T19:43:17.912ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
87892024-11-21T19:43:17.912ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 75.219635ms iteration = 61 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
87902024-11-21T19:43:17.913ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
87912024-11-21T19:43:17.914ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
87922024-11-21T19:43:17.915ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
87932024-11-21T19:43:17.915ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
87942024-11-21T19:43:17.915ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
87952024-11-21T19:43:17.916ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
87962024-11-21T19:43:17.916ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
87972024-11-21T19:43:17.918ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
87982024-11-21T19:43:17.918ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
87992024-11-21T19:43:17.921ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
88002024-11-21T19:43:17.921ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
88012024-11-21T19:43:17.921ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
88022024-11-21T19:43:17.921ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 84.520919ms iteration = 61 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
88032024-11-21T19:43:17.921ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
88042024-11-21T19:43:17.922ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
88052024-11-21T19:43:17.922ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
88062024-11-21T19:43:17.926ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
88072024-11-21T19:43:17.926ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
88082024-11-21T19:43:17.926ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
88092024-11-21T19:43:17.926ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
88102024-11-21T19:43:17.929ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
88112024-11-21T19:43:17.929ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
88122024-11-21T19:43:17.929ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
88132024-11-21T19:43:17.929ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
88142024-11-21T19:43:17.929ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
88152024-11-21T19:43:17.930ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
88162024-11-21T19:43:17.930ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
88172024-11-21T19:43:17.930ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 93.349257ms iteration = 11 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
88182024-11-21T19:43:17.931ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
88192024-11-21T19:43:17.932ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
88202024-11-21T19:43:17.932ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
88212024-11-21T19:43:17.934ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
88222024-11-21T19:43:17.934ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
88232024-11-21T19:43:17.934ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
88242024-11-21T19:43:17.935ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
88252024-11-21T19:43:17.935ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
88262024-11-21T19:43:17.935ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
88272024-11-21T19:43:17.936ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
88282024-11-21T19:43:17.936ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
88292024-11-21T19:43:17.936ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
88302024-11-21T19:43:17.936ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
88312024-11-21T19:43:17.937ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
88322024-11-21T19:43:17.937ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
88332024-11-21T19:43:17.937ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
88342024-11-21T19:43:17.938ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 101.000769ms iteration = 21 status = 0
88352024-11-21T19:43:17.939ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
88362024-11-21T19:43:17.939ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
88372024-11-21T19:43:17.939ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
88382024-11-21T19:43:17.939ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
88392024-11-21T19:43:17.941ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
88402024-11-21T19:43:17.941ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
88412024-11-21T19:43:17.941ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
88422024-11-21T19:43:17.941ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
88432024-11-21T19:43:17.941ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
88442024-11-21T19:43:17.942ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 105.087304ms iteration = 21 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
88452024-11-21T19:43:17.942ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
88462024-11-21T19:43:17.943ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
88472024-11-21T19:43:17.945ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
88482024-11-21T19:43:17.945ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
88492024-11-21T19:43:17.945ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
88502024-11-21T19:43:17.946ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
88512024-11-21T19:43:17.947ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
88522024-11-21T19:43:17.947ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
88532024-11-21T19:43:17.947ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
88542024-11-21T19:43:17.947ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
88552024-11-21T19:43:17.947ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
88562024-11-21T19:43:17.948ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ById(001de000-074c-4000-8000-000000000000) } result = Ok(())
88572024-11-21T19:43:17.948ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
88582024-11-21T19:43:17.949ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
88592024-11-21T19:43:17.949ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
88602024-11-21T19:43:17.953ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
88612024-11-21T19:43:17.953ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
88622024-11-21T19:43:17.953ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
88632024-11-21T19:43:17.972ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
88642024-11-21T19:43:17.972ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
88652024-11-21T19:43:17.992ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
88662024-11-21T19:43:17.992ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
88672024-11-21T19:43:17.992ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 154.623806ms iteration = 21 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
88682024-11-21T19:43:18.012ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-11-21T19:33:15.446964Z, time_modified: 2024-11-21T19:33:15.446964Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }) }
88692024-11-21T19:43:18.012ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 174.853516ms iteration = 11 status = {"by_dns_name":{"default-silo.sys.oxide.test":{"silo_id":"001de000-5110-4000-8000-000000000000","tls_certs":[]},"recovery.sys.oxide.test":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]}},"default_endpoint":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]},"warnings":["silo 001de000-5110-4000-8000-000000000000 with DNS name \\"default-silo.sys.oxide.test\\" has no usable certificates"]}
88702024-11-21T19:43:18.016ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
88712024-11-21T19:43:18.025ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
88722024-11-21T19:43:18.037ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 200.512159ms iteration = 22 status = {"physical_disks_added":0}
88732024-11-21T19:43:18.046ZWARNnexus (BackgroundTasks): instance reincarnation completed with errors background_task = instance_reincarnation file = nexus/src/app/background/tasks/instance_reincarnation.rs:103 instances_changed_state = 0 instances_found = 0 instances_reincarnated = 0 query_errors = 0 restart_errors = 0
88742024-11-21T19:43:18.046ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_reincarnation elapsed = 208.394054ms iteration = 11 status = {"changed_state":[],"disabled":false,"errors":[],"instances_found":{"Failed":0,"SagaUnwound":0},"instances_reincarnated":[],"restart_errors":[]}
88752024-11-21T19:43:18.051ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
88762024-11-21T19:43:18.051ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
88772024-11-21T19:43:18.052ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "5f384b01-3419-461d-a146-fc59b4a962ea", "content-length": "26", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
88782024-11-21T19:43:18.052ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
88792024-11-21T19:43:18.053ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
88802024-11-21T19:43:18.106ZDEBGnexus (BackgroundTasks): begin collection background_task = inventory_collection
88812024-11-21T19:43:18.106ZDEBGnexus (BackgroundTasks): begin collection from MGS background_task = inventory_collection mgs_url = http://[fd00:1122:3344:101::2]:12225
88822024-11-21T19:43:18.106ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/ignition
88832024-11-21T19:43:18.106ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/ignition", status: 200, headers: {"content-type": "application/json", "x-request-id": "a0033503-7e98-4d1b-8145-9ee2a8e40641", "content-length": "476", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
88842024-11-21T19:43:18.106ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0
88852024-11-21T19:43:18.106ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0", status: 200, headers: {"content-type": "application/json", "x-request-id": "6b2475e1-9acc-4cb4-ad4e-0a4e2f5c28de", "content-length": "734", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
88862024-11-21T19:43:18.106ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/sp/caboose?firmware_slot=0
88872024-11-21T19:43:18.107ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/sp/caboose?firmware_slot=0", status: 200, headers: {"content-type": "application/json", "x-request-id": "c766248a-d738-4863-a340-98dd83faeaa8", "content-length": "111", "date": "Thu, 21 Nov 2024 19:43:17 GMT"} })
88882024-11-21T19:43:18.107ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/sp/caboose?firmware_slot=1
88892024-11-21T19:43:18.108ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/sp/caboose?firmware_slot=1", status: 200, headers: {"content-type": "application/json", "x-request-id": "93fc7113-dac0-41f0-ad63-40a904948151", "content-length": "111", "date": "Thu, 21 Nov 2024 19:43:17 GMT"} })
88902024-11-21T19:43:18.108ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/caboose?firmware_slot=0
88912024-11-21T19:43:18.108ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/caboose?firmware_slot=0", status: 200, headers: {"content-type": "application/json", "x-request-id": "b2a25e22-7f72-4688-9df7-1f574b5004cd", "content-length": "167", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
88922024-11-21T19:43:18.108ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/caboose?firmware_slot=1
88932024-11-21T19:43:18.109ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/caboose?firmware_slot=1", status: 200, headers: {"content-type": "application/json", "x-request-id": "67f1b260-c8eb-4b96-b7ec-b6c8dd424e27", "content-length": "167", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
88942024-11-21T19:43:18.109ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/stage0/caboose?firmware_slot=0
88952024-11-21T19:43:18.109ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/stage0/caboose?firmware_slot=0", status: 200, headers: {"content-type": "application/json", "x-request-id": "ef8c363a-8d78-4880-ad25-549aead2c1d2", "content-length": "175", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
88962024-11-21T19:43:18.109ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/stage0/caboose?firmware_slot=1
88972024-11-21T19:43:18.109ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/stage0/caboose?firmware_slot=1", status: 200, headers: {"content-type": "application/json", "x-request-id": "ea1f8f39-7ad7-4fe4-8c45-05c811bd329c", "content-length": "175", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
88982024-11-21T19:43:18.109ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/cmpa
88992024-11-21T19:43:18.110ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/cmpa", status: 200, headers: {"content-type": "application/json", "x-request-id": "93389b11-3674-47ed-bd3c-59e803d358a7", "content-length": "702", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
89002024-11-21T19:43:18.110ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = Some(Body) gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/cfpa
89012024-11-21T19:43:18.110ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/cfpa", status: 200, headers: {"content-type": "application/json", "x-request-id": "d6b716e9-75cd-40ee-b1a3-290d6b90d302", "content-length": "727", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
89022024-11-21T19:43:18.110ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = Some(Body) gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/cfpa
89032024-11-21T19:43:18.110ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/cfpa", status: 200, headers: {"content-type": "application/json", "x-request-id": "82c6f487-f952-4b46-a3de-beeb010f1281", "content-length": "729", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
89042024-11-21T19:43:18.110ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = Some(Body) gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/cfpa
89052024-11-21T19:43:18.110ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/switch/0/component/rot/cfpa", status: 200, headers: {"content-type": "application/json", "x-request-id": "27459a62-77b0-428c-9916-8da6a0cbd737", "content-length": "728", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
89062024-11-21T19:43:18.110ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0
89072024-11-21T19:43:18.110ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0", status: 200, headers: {"content-type": "application/json", "x-request-id": "dc5785f8-4fa0-44bc-ab7b-d31be63dfd75", "content-length": "722", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
89082024-11-21T19:43:18.110ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/sp/caboose?firmware_slot=0
89092024-11-21T19:43:18.111ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/sp/caboose?firmware_slot=0", status: 200, headers: {"content-type": "application/json", "x-request-id": "b3ba5d6c-7416-4428-80d1-99717a5ac129", "content-length": "109", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
89102024-11-21T19:43:18.111ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/sp/caboose?firmware_slot=1
89112024-11-21T19:43:18.111ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/sp/caboose?firmware_slot=1", status: 200, headers: {"content-type": "application/json", "x-request-id": "509d1584-6f19-4d0d-849c-f4ffc10e7395", "content-length": "109", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
89122024-11-21T19:43:18.111ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/caboose?firmware_slot=0
89132024-11-21T19:43:18.112ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/caboose?firmware_slot=0", status: 200, headers: {"content-type": "application/json", "x-request-id": "403838e4-4ec1-4ad7-8c05-a86fdce300d8", "content-length": "169", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
89142024-11-21T19:43:18.112ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/caboose?firmware_slot=1
89152024-11-21T19:43:18.112ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/caboose?firmware_slot=1", status: 200, headers: {"content-type": "application/json", "x-request-id": "f67740ae-9f03-4d22-8a6c-784a3af96347", "content-length": "169", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
89162024-11-21T19:43:18.112ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/stage0/caboose?firmware_slot=0
89172024-11-21T19:43:18.113ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/stage0/caboose?firmware_slot=0", status: 200, headers: {"content-type": "application/json", "x-request-id": "e7e82a5b-3a2c-4721-982c-230f99ddc4f9", "content-length": "178", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
89182024-11-21T19:43:18.113ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/stage0/caboose?firmware_slot=1
89192024-11-21T19:43:18.113ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/stage0/caboose?firmware_slot=1", status: 200, headers: {"content-type": "application/json", "x-request-id": "0a6c7cf5-43aa-4b6d-b476-a670679840ff", "content-length": "178", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
89202024-11-21T19:43:18.113ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = None gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/cmpa
89212024-11-21T19:43:18.114ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/cmpa", status: 200, headers: {"content-type": "application/json", "x-request-id": "6f800c27-f224-4181-8fd5-9b97713cd9e7", "content-length": "702", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
89222024-11-21T19:43:18.114ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = Some(Body) gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/cfpa
89232024-11-21T19:43:18.114ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/cfpa", status: 200, headers: {"content-type": "application/json", "x-request-id": "a4854474-70a6-4bd9-b343-a854f743136d", "content-length": "727", "date": "Thu, 21 Nov 2024 19:43:17 GMT"} })
89242024-11-21T19:43:18.114ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = Some(Body) gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/cfpa
89252024-11-21T19:43:18.114ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/cfpa", status: 200, headers: {"content-type": "application/json", "x-request-id": "f0fd0873-1ccf-48d6-b44d-4d4b0a276400", "content-length": "729", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
89262024-11-21T19:43:18.114ZDEBGnexus (BackgroundTasks): client request background_task = inventory_collection body = Some(Body) gateway_url = http://[fd00:1122:3344:101::2]:12225 method = GET uri = http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/cfpa
89272024-11-21T19:43:18.114ZDEBGnexus (BackgroundTasks): client response background_task = inventory_collection gateway_url = http://[fd00:1122:3344:101::2]:12225 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/sp/sled/0/component/rot/cfpa", status: 200, headers: {"content-type": "application/json", "x-request-id": "9715d8c5-84f1-4cd1-a256-61bf6bc95af6", "content-length": "728", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
89282024-11-21T19:43:18.114ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {} }
89292024-11-21T19:43:18.115ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Database result = Ok(())
89302024-11-21T19:43:18.178ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
89312024-11-21T19:43:18.178ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
89322024-11-21T19:43:18.179ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "4dce7041-9b62-40da-a8cb-e6d7731b1abd", "content-length": "26", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
89332024-11-21T19:43:18.179ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
89342024-11-21T19:43:18.180ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
89352024-11-21T19:43:18.180ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "7517455b-fc32-4062-ab45-7b2673809fda", "content-length": "691", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
89362024-11-21T19:43:18.180ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
89372024-11-21T19:43:18.180ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
89382024-11-21T19:43:18.180ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
89392024-11-21T19:43:18.181ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
89402024-11-21T19:43:18.182ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
89412024-11-21T19:43:18.183ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
89422024-11-21T19:43:18.183ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 346.793323ms iteration = 21 status = {}
89432024-11-21T19:43:18.184ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 346.305317ms iteration = 21 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
89442024-11-21T19:43:18.184ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
89452024-11-21T19:43:18.184ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
89462024-11-21T19:43:18.184ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 51 version = 1
89472024-11-21T19:43:18.184ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
89482024-11-21T19:43:18.184ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-11-21 19:33:31.797390 UTC dns_group = internal generation = 1
89492024-11-21T19:43:18.184ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 347.485613ms iteration = 11 status = {"generation":1}
89502024-11-21T19:43:18.184ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
89512024-11-21T19:43:18.186ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Fleet result = Ok(())
89522024-11-21T19:43:18.186ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {} }
89532024-11-21T19:43:18.186ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Database result = Ok(())
89542024-11-21T19:43:18.187ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
89552024-11-21T19:43:18.187ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
89562024-11-21T19:43:18.220ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
89572024-11-21T19:43:18.220ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
89582024-11-21T19:43:18.220ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "afde6520-8fa5-46d2-b2a8-96a60da6bf85", "content-length": "26", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
89592024-11-21T19:43:18.220ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
89602024-11-21T19:43:18.222ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
89612024-11-21T19:43:18.237ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
89622024-11-21T19:43:18.238ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "852c3c4f-5969-40b1-8fac-189a9e16d87f", "content-length": "1", "date": "Thu, 21 Nov 2024 19:43:17 GMT"} })
89632024-11-21T19:43:18.239ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
89642024-11-21T19:43:18.240ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
89652024-11-21T19:43:18.251ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
89662024-11-21T19:43:18.252ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::8]:5353/config", status: 204, headers: {"x-request-id": "67b7eca9-6af0-4370-a31c-429c4d67aa56", "date": "Thu, 21 Nov 2024 19:43:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
89672024-11-21T19:43:18.252ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
89682024-11-21T19:43:18.254ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::9]:5353/config", status: 204, headers: {"x-request-id": "46825495-02ef-4a3a-83a8-9bdeff681583", "date": "Thu, 21 Nov 2024 19:43:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
89692024-11-21T19:43:18.254ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
89702024-11-21T19:43:18.255ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
89712024-11-21T19:43:18.255ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 418.802976ms iteration = 13 status = {"generation":2,"server_results":{"[fd00:1122:3344:101::8]:5353":{"Ok":null},"[fd00:1122:3344:101::9]:5353":{"Ok":null}}}
89722024-11-21T19:43:18.311ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
89732024-11-21T19:43:18.311ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
89742024-11-21T19:43:18.313ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
89752024-11-21T19:43:18.315ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 478.639515ms iteration = 21 status = null
89762024-11-21T19:43:18.315ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
89772024-11-21T19:43:18.316ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
89782024-11-21T19:43:18.317ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Fleet result = Ok(())
89792024-11-21T19:43:18.317ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {} }
89802024-11-21T19:43:18.317ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Database result = Ok(())
89812024-11-21T19:43:18.318ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
89822024-11-21T19:43:18.318ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
89832024-11-21T19:43:18.319ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
89842024-11-21T19:43:18.320ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
89852024-11-21T19:43:18.320ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
89862024-11-21T19:43:18.322ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
89872024-11-21T19:43:18.326ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000002, lookup_type: ByName("nexus") } result = Ok(())
89882024-11-21T19:43:18.326ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
89892024-11-21T19:43:18.326ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
89902024-11-21T19:43:18.328ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
89912024-11-21T19:43:18.328ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 491.795211ms iteration = 21 status = {}
89922024-11-21T19:43:18.328ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
89932024-11-21T19:43:18.330ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
89942024-11-21T19:43:18.331ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
89952024-11-21T19:43:18.343ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "45c6514d-46af-49c4-8d1a-83d49052b9c6", "content-length": "372", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
89962024-11-21T19:43:18.343ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
89972024-11-21T19:43:18.344ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
89982024-11-21T19:43:18.426ZDEBGnexus (BackgroundTasks): begin collection from Sled Agent background_task = inventory_collection sled_agent_url = http://[fd00:1122:3344:101::1]:12345
89992024-11-21T19:43:18.426ZDEBGnexus (BackgroundTasks): client request SledAgent = http://[fd00:1122:3344:101::1]:12345 background_task = inventory_collection body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/inventory
90002024-11-21T19:43:18.428ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
90012024-11-21T19:43:18.430ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
90022024-11-21T19:43:18.430ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:1::1]:5353/config", status: 204, headers: {"x-request-id": "891936c5-e724-4639-b7a6-0c349c54e409", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
90032024-11-21T19:43:18.430ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
90042024-11-21T19:43:18.432ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:3::1]:5353/config", status: 204, headers: {"x-request-id": "06618efa-32e5-4d8d-bd57-58d469f3bcae", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
90052024-11-21T19:43:18.432ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
90062024-11-21T19:43:18.433ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000002, lookup_type: ByName("nexus") } result = Ok(())
90072024-11-21T19:43:18.433ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
90082024-11-21T19:43:18.433ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
90092024-11-21T19:43:18.434ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:2::1]:5353/config", status: 204, headers: {"x-request-id": "cce05c5d-6478-46a2-a922-3446f116cd57", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
90102024-11-21T19:43:18.434ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
90112024-11-21T19:43:18.437ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
90122024-11-21T19:43:18.437ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 600.2056ms iteration = 13 status = {"generation":1,"server_results":{"[fd00:1122:3344:1::1]:5353":{"Ok":null},"[fd00:1122:3344:2::1]:5353":{"Ok":null},"[fd00:1122:3344:3::1]:5353":{"Ok":null}}}
90132024-11-21T19:43:18.439ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
90142024-11-21T19:43:18.439ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
90152024-11-21T19:43:18.442ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
90162024-11-21T19:43:18.442ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
90172024-11-21T19:43:18.443ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
90182024-11-21T19:43:18.444ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
90192024-11-21T19:43:18.444ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
90202024-11-21T19:43:18.444ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
90212024-11-21T19:43:18.447ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
90222024-11-21T19:43:18.447ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
90232024-11-21T19:43:18.450ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
90242024-11-21T19:43:18.450ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
90252024-11-21T19:43:18.452ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "6c6ade3b-4175-4e91-8da5-920536f0c0d8", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
90262024-11-21T19:43:18.452ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
90272024-11-21T19:43:18.452ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
90282024-11-21T19:43:18.452ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
90292024-11-21T19:43:18.452ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
90302024-11-21T19:43:18.454ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
90312024-11-21T19:43:18.455ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
90322024-11-21T19:43:18.455ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
90332024-11-21T19:43:18.455ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
90342024-11-21T19:43:18.457ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
90352024-11-21T19:43:18.458ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
90362024-11-21T19:43:18.458ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
90372024-11-21T19:43:18.461ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000001, lookup_type: ByName("external-dns") } result = Ok(())
90382024-11-21T19:43:18.461ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
90392024-11-21T19:43:18.461ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
90402024-11-21T19:43:18.462ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
90412024-11-21T19:43:18.462ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
90422024-11-21T19:43:18.463ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
90432024-11-21T19:43:18.463ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
90442024-11-21T19:43:18.465ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
90452024-11-21T19:43:18.465ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
90462024-11-21T19:43:18.467ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
90472024-11-21T19:43:18.471ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = VpcSubnet { parent: Vpc { parent: Project { parent: Silo { parent: Fleet, key: 001de000-5110-4000-8000-000000000001, lookup_type: ById(001de000-5110-4000-8000-000000000001) }, key: 001de000-4401-4000-8000-000000000000, lookup_type: ById(001de000-4401-4000-8000-000000000000) }, key: 001de000-074c-4000-8000-000000000000, lookup_type: ByName("oxide-services") }, key: 001de000-c470-4000-8000-000000000001, lookup_type: ByName("external-dns") } result = Ok(())
90482024-11-21T19:43:18.472ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
90492024-11-21T19:43:18.472ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
90502024-11-21T19:43:18.477ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "a8a265c8-3db3-4edd-a237-3754a2997b8d", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
90512024-11-21T19:43:18.479ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 641.489941ms iteration = 21 status = {}
90522024-11-21T19:43:18.479ZDEBGnexus (service-firewall-rule-progator): resolved names for firewall rules background_task = service_firewall_rule_propagation instance_interfaces = {} subnet_interfaces = {Name("external-dns"): [NetworkInterface { id: 4c211da9-c36c-41bc-8274-9aa6420c20e6, kind: Service { id: 90b200e8-d1ee-4e0d-975c-64628c5532d2 }, name: Name("external-dns-90b200e8-d1ee-4e0d-975c-64628c5532d2"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 185])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b69bf686-c443-40c6-b854-c2ff6e5e107a, kind: Service { id: 4e669ff4-6dde-4cd2-8491-623460b56d8d }, name: Name("external-dns-4e669ff4-6dde-4cd2-8491-623460b56d8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 188])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], Name("nexus"): [NetworkInterface { id: 03ccecdf-44ea-4815-8920-5ebe8d7bf7ff, kind: Service { id: 915e11b8-d7cb-4730-9906-7980540813dc }, name: Name("nexus-915e11b8-d7cb-4730-9906-7980540813dc"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 146])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 4cb2fda6-91fb-43d0-be7c-77f85347c432, kind: Service { id: 39d14e41-6d5d-474a-aaea-65c36b8f9da5 }, name: Name("nexus-39d14e41-6d5d-474a-aaea-65c36b8f9da5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 243])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7e0b7555-efc0-41d1-b3ba-7ca41e2e30b7, kind: Service { id: 088dbd0f-363c-4785-9d04-aa968256294e }, name: Name("nexus-088dbd0f-363c-4785-9d04-aa968256294e"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 205])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }]} subnet_networks = {Name("external-dns"): [V4(Ipv4Network { addr: 172.30.1.0, prefix: 24 }), V6(Ipv6Network { addr: fd77:e9d2:9cd9:1::, prefix: 64 })], Name("nexus"): [V4(Ipv4Network { addr: 172.30.2.0, prefix: 24 }), V6(Ipv6Network { addr: fd77:e9d2:9cd9:2::, prefix: 64 })]} vpc_interfaces = {}
90532024-11-21T19:43:18.479ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
90542024-11-21T19:43:18.479ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
90552024-11-21T19:43:18.480ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
90562024-11-21T19:43:18.480ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
90572024-11-21T19:43:18.481ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
90582024-11-21T19:43:18.482ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
90592024-11-21T19:43:18.482ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
90602024-11-21T19:43:18.482ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
90612024-11-21T19:43:18.483ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Fleet result = Ok(())
90622024-11-21T19:43:18.483ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
90632024-11-21T19:43:18.483ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
90642024-11-21T19:43:18.485ZDEBGnexus (service-firewall-rule-progator): Allowlist for user-facing services is set to allow any inbound traffic. Existing VPC firewall rules will not be modified. background_task = service_firewall_rule_propagation
90652024-11-21T19:43:18.485ZDEBGnexus (service-firewall-rule-progator): resolved firewall rules for sled agents background_task = service_firewall_rule_propagation sled_agent_rules = [ResolvedVpcFirewallRule { action: Allow, direction: Inbound, filter_hosts: None, filter_ports: Some([L4PortRange("53")]), filter_protocols: Some([Udp]), priority: 65534, status: Enabled, targets: [NetworkInterface { id: 4c211da9-c36c-41bc-8274-9aa6420c20e6, kind: Service { id: 90b200e8-d1ee-4e0d-975c-64628c5532d2 }, name: Name("external-dns-90b200e8-d1ee-4e0d-975c-64628c5532d2"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 185])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b69bf686-c443-40c6-b854-c2ff6e5e107a, kind: Service { id: 4e669ff4-6dde-4cd2-8491-623460b56d8d }, name: Name("external-dns-4e669ff4-6dde-4cd2-8491-623460b56d8d"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 188])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }] }, ResolvedVpcFirewallRule { action: Allow, direction: Inbound, filter_hosts: None, filter_ports: Some([L4PortRange("80"), L4PortRange("443")]), filter_protocols: Some([Tcp]), priority: 65534, status: Enabled, targets: [NetworkInterface { id: 03ccecdf-44ea-4815-8920-5ebe8d7bf7ff, kind: Service { id: 915e11b8-d7cb-4730-9906-7980540813dc }, name: Name("nexus-915e11b8-d7cb-4730-9906-7980540813dc"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 146])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 4cb2fda6-91fb-43d0-be7c-77f85347c432, kind: Service { id: 39d14e41-6d5d-474a-aaea-65c36b8f9da5 }, name: Name("nexus-39d14e41-6d5d-474a-aaea-65c36b8f9da5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 243])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7e0b7555-efc0-41d1-b3ba-7ca41e2e30b7, kind: Service { id: 088dbd0f-363c-4785-9d04-aa968256294e }, name: Name("nexus-088dbd0f-363c-4785-9d04-aa968256294e"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 205])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }] }]
90662024-11-21T19:43:18.485ZDEBGnexus (service-firewall-rule-progator): resolved 2 rules for sleds background_task = service_firewall_rule_propagation
90672024-11-21T19:43:18.490ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
90682024-11-21T19:43:18.491ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "36702f63-e50c-48b7-98b7-8df684b9e7dc", "content-length": "134", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
90692024-11-21T19:43:18.491ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
90702024-11-21T19:43:18.491ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
90712024-11-21T19:43:18.491ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
90722024-11-21T19:43:18.491ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
90732024-11-21T19:43:18.491ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
90742024-11-21T19:43:18.491ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
90752024-11-21T19:43:18.491ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
90762024-11-21T19:43:18.491ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
90772024-11-21T19:43:18.491ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
90782024-11-21T19:43:18.491ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
90792024-11-21T19:43:18.491ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
90802024-11-21T19:43:18.491ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
90812024-11-21T19:43:18.491ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
90822024-11-21T19:43:18.494ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
90832024-11-21T19:43:18.494ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "f0b978e8-dd04-4bfa-a07e-7355776688bb", "content-length": "2", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
90842024-11-21T19:43:18.494ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
90852024-11-21T19:43:18.494ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "543e71e1-264e-408f-86bb-fd87e3d7b35f", "content-length": "51", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
90862024-11-21T19:43:18.494ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
90872024-11-21T19:43:18.494ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
90882024-11-21T19:43:18.547ZDEBGnexus (service-firewall-rule-progator): resolved sleds for vpc oxide-services background_task = service_firewall_rule_propagation vpc_to_sled = [Sled { identity: SledIdentity { id: 203795ed-576b-40e4-bed8-7463e3f2162c, time_created: 2024-11-21T19:33:30.261358Z, time_modified: 2024-11-21T19:33:30.261358Z }, time_deleted: None, rcgen: Generation(Generation(19)), rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, is_scrimlet: true, serial_number: "buskin", part_number: "i86pc", revision: SqlU32(0), usable_hardware_threads: SqlU32(48), usable_physical_ram: ByteCount(ByteCount(102952026112)), reservoir_size: ByteCount(ByteCount(51474595840)), ip: fd00:1122:3344:101::1, port: SqlU16(12345), last_used_address: fd00:1122:3344:101::ffff, policy: InService, state: Active, sled_agent_gen: Generation(Generation(1)) }]
90892024-11-21T19:43:18.547ZDEBGnexus (service-firewall-rule-progator): sending firewall rules to sled agents background_task = service_firewall_rule_propagation
90902024-11-21T19:43:18.547ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
90912024-11-21T19:43:18.548ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
90922024-11-21T19:43:18.550ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {} }
90932024-11-21T19:43:18.550ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Database result = Ok(())
90942024-11-21T19:43:18.552ZDEBGnexus (BackgroundTasks): roles background_task = service_firewall_rule_propagation roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
90952024-11-21T19:43:18.554ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_firewall_rule_propagation resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
90962024-11-21T19:43:18.587ZDEBGnexus (service-firewall-rule-progator): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = service_firewall_rule_propagation body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc/001de000-074c-4000-8000-000000000000/firewall/rules
90972024-11-21T19:43:18.592ZDEBGnexus (service-firewall-rule-progator): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = service_firewall_rule_propagation result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc/001de000-074c-4000-8000-000000000000/firewall/rules", status: 204, headers: {"x-request-id": "6a56c588-8d29-4fc3-9a1a-f0d83366d07a", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
90982024-11-21T19:43:18.595ZINFOnexus (service-firewall-rule-progator): updated firewall rules on 1 sleds background_task = service_firewall_rule_propagation file = nexus/networking/src/firewall_rules.rs:496
90992024-11-21T19:43:18.595ZDEBGnexus (service-firewall-rule-progator): successfully propagated service firewall rules background_task = service_firewall_rule_propagation elapsed = 757.260768ms
91002024-11-21T19:43:18.595ZDEBGnexus (BackgroundTasks): activation complete background_task = service_firewall_rule_propagation elapsed = 757.268198ms iteration = 3 status = {}
91012024-11-21T19:43:18.637ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "3646d0a7-98f6-4c9a-bc1f-7a551647d179", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
91022024-11-21T19:43:18.637ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
91032024-11-21T19:43:18.637ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
91042024-11-21T19:43:18.637ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "8bc1985f-63fc-4d42-b83b-95dba6a26340", "content-length": "682", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
91052024-11-21T19:43:18.637ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
91062024-11-21T19:43:18.638ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
91072024-11-21T19:43:18.641ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
91082024-11-21T19:43:18.641ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
91092024-11-21T19:43:18.644ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
91102024-11-21T19:43:18.644ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
91112024-11-21T19:43:18.646ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
91122024-11-21T19:43:18.646ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
91132024-11-21T19:43:18.653ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 816.483559ms iteration = 21 status = {}
91142024-11-21T19:43:18.892ZDEBGnexus (BackgroundTasks): client response SledAgent = http://[fd00:1122:3344:101::1]:12345 background_task = inventory_collection result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/inventory", status: 200, headers: {"content-type": "application/json", "x-request-id": "89133fdd-9e7c-4105-b856-352c2c506dc5", "content-length": "30761", "date": "Thu, 21 Nov 2024 19:43:18 GMT"} })
91152024-11-21T19:43:18.913ZDEBGnexus (BackgroundTasks): begin collecting all keepers background_task = inventory_collection nkeeper_admin_clients = 0
91162024-11-21T19:43:18.913ZDEBGnexus (BackgroundTasks): end collecting all keepers background_task = inventory_collection nkeeper_admin_clients = 0
91172024-11-21T19:43:18.913ZDEBGnexus (BackgroundTasks): finished collection background_task = inventory_collection
91182024-11-21T19:43:18.915ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {} }
91192024-11-21T19:43:18.915ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Database result = Ok(())
91202024-11-21T19:43:18.917ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
91212024-11-21T19:43:18.919ZDEBGnexus (BackgroundTasks): authorize result action = Modify actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Inventory result = Ok(())
91222024-11-21T19:43:18.922ZDEBGnexus (BackgroundTasks): roles background_task = inventory_collection roles = RoleSet { roles: {} }
91232024-11-21T19:43:18.922ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = inventory_collection resource = Database result = Ok(())
91242024-11-21T19:43:19.118ZINFOnexus (BackgroundTasks): inserted inventory collection background_task = inventory_collection collection_id = a2aa78a8-d920-4f76-af99-5298625ec48d file = nexus/db-queries/src/db/datastore/inventory.rs:1000
91252024-11-21T19:43:19.118ZDEBGnexus (BackgroundTasks): inventory collection complete background_task = inventory_collection collection_id = a2aa78a8-d920-4f76-af99-5298625ec48d time_started = 2024-11-21 19:43:18.105998 UTC
91262024-11-21T19:43:19.118ZDEBGnexus (BackgroundTasks): activation complete background_task = inventory_collection elapsed = 1.281882097s iteration = 2 status = {"collection_id":"a2aa78a8-d920-4f76-af99-5298625ec48d","time_done":"2024-11-21 19:43:18.913331 UTC","time_started":"2024-11-21 19:43:18.105998 UTC"}
91272024-11-21T19:43:19.118ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 23 reason = Dependency
91282024-11-21T19:43:19.119ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
91292024-11-21T19:43:19.119ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
91302024-11-21T19:43:19.120ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
91312024-11-21T19:43:19.122ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
91322024-11-21T19:43:19.122ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
91332024-11-21T19:43:19.122ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
91342024-11-21T19:43:19.125ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
91352024-11-21T19:43:19.125ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
91362024-11-21T19:43:19.127ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
91372024-11-21T19:43:19.129ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
91382024-11-21T19:43:19.129ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
91392024-11-21T19:43:19.129ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
91402024-11-21T19:43:19.163ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 44.765458ms iteration = 23 status = {"physical_disks_added":0}
91412024-11-21T19:43:26.964ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 559 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 29ee7ed4-b21f-4d40-92e2-6d29eb263644 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
91422024-11-21T19:43:27.837ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 62 reason = Timeout
91432024-11-21T19:43:27.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
91442024-11-21T19:43:27.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
91452024-11-21T19:43:27.838ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 62 reason = Timeout
91462024-11-21T19:43:27.838ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
91472024-11-21T19:43:27.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
91482024-11-21T19:43:27.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
91492024-11-21T19:43:27.841ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
91502024-11-21T19:43:27.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
91512024-11-21T19:43:27.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
91522024-11-21T19:43:27.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
91532024-11-21T19:43:27.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
91542024-11-21T19:43:27.843ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 4.636082ms iteration = 62 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
91552024-11-21T19:43:27.844ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
91562024-11-21T19:43:27.844ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
91572024-11-21T19:43:27.845ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
91582024-11-21T19:43:27.848ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
91592024-11-21T19:43:27.848ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
91602024-11-21T19:43:27.848ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
91612024-11-21T19:43:27.857ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
91622024-11-21T19:43:27.857ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 19.292398ms iteration = 62 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
91632024-11-21T19:43:37.837ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 63 reason = Timeout
91642024-11-21T19:43:37.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
91652024-11-21T19:43:37.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
91662024-11-21T19:43:37.838ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 63 reason = Timeout
91672024-11-21T19:43:37.838ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
91682024-11-21T19:43:37.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
91692024-11-21T19:43:37.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
91702024-11-21T19:43:37.840ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
91712024-11-21T19:43:37.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
91722024-11-21T19:43:37.842ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 4.014803ms iteration = 63 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
91732024-11-21T19:43:37.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
91742024-11-21T19:43:37.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
91752024-11-21T19:43:37.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
91762024-11-21T19:43:37.844ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
91772024-11-21T19:43:37.844ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
91782024-11-21T19:43:37.846ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
91792024-11-21T19:43:37.848ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
91802024-11-21T19:43:37.848ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
91812024-11-21T19:43:37.848ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
91822024-11-21T19:43:37.857ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
91832024-11-21T19:43:37.857ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 19.838285ms iteration = 63 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
91842024-11-21T19:43:41.966ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 339 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 3bd64c41-839c-4485-b9a8-ddede9353f3c response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
91852024-11-21T19:43:47.837ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 22 reason = Timeout
91862024-11-21T19:43:47.837ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
91872024-11-21T19:43:47.837ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 64 reason = Timeout
91882024-11-21T19:43:47.837ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 22 reason = Timeout
91892024-11-21T19:43:47.837ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
91902024-11-21T19:43:47.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
91912024-11-21T19:43:47.837ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 24 reason = Timeout
91922024-11-21T19:43:47.837ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
91932024-11-21T19:43:47.837ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 22 reason = Timeout
91942024-11-21T19:43:47.837ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
91952024-11-21T19:43:47.837ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 22 reason = Timeout
91962024-11-21T19:43:47.837ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 22 reason = Timeout
91972024-11-21T19:43:47.837ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 22 reason = Timeout
91982024-11-21T19:43:47.837ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
91992024-11-21T19:43:47.837ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
92002024-11-21T19:43:47.837ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 22 reason = Timeout
92012024-11-21T19:43:47.837ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
92022024-11-21T19:43:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
92032024-11-21T19:43:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
92042024-11-21T19:43:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
92052024-11-21T19:43:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
92062024-11-21T19:43:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
92072024-11-21T19:43:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
92082024-11-21T19:43:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
92092024-11-21T19:43:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
92102024-11-21T19:43:47.838ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 64 reason = Timeout
92112024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
92122024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 22 reason = Timeout
92132024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
92142024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 22 reason = Timeout
92152024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
92162024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 22 reason = Timeout
92172024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
92182024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
92192024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 22 reason = Timeout
92202024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
92212024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 22 reason = Timeout
92222024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
92232024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 22 reason = Timeout
92242024-11-21T19:43:47.839ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
92252024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 22 reason = Timeout
92262024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
92272024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
92282024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
92292024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
92302024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
92312024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
92322024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
92332024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
92342024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
92352024-11-21T19:43:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
92362024-11-21T19:43:47.840ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 2.737296ms iteration = 22 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
92372024-11-21T19:43:47.840ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1370, tv_nsec: 711907455 } })
92382024-11-21T19:43:47.841ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
92392024-11-21T19:43:47.841ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
92402024-11-21T19:43:47.841ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1370, tv_nsec: 712550774 } })
92412024-11-21T19:43:47.841ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
92422024-11-21T19:43:47.841ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
92432024-11-21T19:43:47.842ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
92442024-11-21T19:43:47.844ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
92452024-11-21T19:43:47.844ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
92462024-11-21T19:43:47.844ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
92472024-11-21T19:43:47.844ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
92482024-11-21T19:43:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
92492024-11-21T19:43:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
92502024-11-21T19:43:47.845ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
92512024-11-21T19:43:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
92522024-11-21T19:43:47.845ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
92532024-11-21T19:43:47.846ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
92542024-11-21T19:43:47.846ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
92552024-11-21T19:43:47.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
92562024-11-21T19:43:47.847ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
92572024-11-21T19:43:47.847ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
92582024-11-21T19:43:47.847ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
92592024-11-21T19:43:47.848ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 9.426006ms iteration = 22 status = {"errors":[],"garbage_collect_requested":[]}
92602024-11-21T19:43:47.849ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 10.189116ms iteration = 22 status = {"errors":[],"records_set_to_done":[]}
92612024-11-21T19:43:47.849ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
92622024-11-21T19:43:47.850ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
92632024-11-21T19:43:47.850ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
92642024-11-21T19:43:47.850ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
92652024-11-21T19:43:47.851ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
92662024-11-21T19:43:47.851ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
92672024-11-21T19:43:47.851ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
92682024-11-21T19:43:47.851ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
92692024-11-21T19:43:47.852ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
92702024-11-21T19:43:47.852ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
92712024-11-21T19:43:47.852ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
92722024-11-21T19:43:47.852ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
92732024-11-21T19:43:47.852ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
92742024-11-21T19:43:47.852ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
92752024-11-21T19:43:47.853ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
92762024-11-21T19:43:47.853ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
92772024-11-21T19:43:47.853ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
92782024-11-21T19:43:47.853ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 16.103975ms iteration = 22 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
92792024-11-21T19:43:47.854ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
92802024-11-21T19:43:47.854ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
92812024-11-21T19:43:47.854ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
92822024-11-21T19:43:47.855ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
92832024-11-21T19:43:47.855ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
92842024-11-21T19:43:47.855ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
92852024-11-21T19:43:47.855ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
92862024-11-21T19:43:47.855ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1370, tv_nsec: 726675353 } })
92872024-11-21T19:43:47.855ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
92882024-11-21T19:43:47.856ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 17.220331ms iteration = 22 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
92892024-11-21T19:43:47.857ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
92902024-11-21T19:43:47.858ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
92912024-11-21T19:43:47.859ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
92922024-11-21T19:43:47.859ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
92932024-11-21T19:43:47.859ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
92942024-11-21T19:43:47.859ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
92952024-11-21T19:43:47.859ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
92962024-11-21T19:43:47.860ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
92972024-11-21T19:43:47.860ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
92982024-11-21T19:43:47.860ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
92992024-11-21T19:43:47.860ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
93002024-11-21T19:43:47.860ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 21.71687ms iteration = 22 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
93012024-11-21T19:43:47.861ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
93022024-11-21T19:43:47.861ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
93032024-11-21T19:43:47.862ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
93042024-11-21T19:43:47.863ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
93052024-11-21T19:43:47.863ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
93062024-11-21T19:43:47.863ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
93072024-11-21T19:43:47.873ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
93082024-11-21T19:43:47.873ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 35.589185ms iteration = 64 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
93092024-11-21T19:43:47.877ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
93102024-11-21T19:43:47.877ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
93112024-11-21T19:43:47.881ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
93122024-11-21T19:43:47.881ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
93132024-11-21T19:43:47.891ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 52.434641ms iteration = 64 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
93142024-11-21T19:43:47.894ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
93152024-11-21T19:43:47.894ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
93162024-11-21T19:43:47.894ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 55.505061ms iteration = 22 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
93172024-11-21T19:43:47.903ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 65.299942ms iteration = 24 status = {"physical_disks_added":0}
93182024-11-21T19:43:47.906ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
93192024-11-21T19:43:47.907ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
93202024-11-21T19:43:47.913ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
93212024-11-21T19:43:47.914ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
93222024-11-21T19:43:47.941ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
93232024-11-21T19:43:47.941ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
93242024-11-21T19:43:47.941ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "0defa73a-914a-478d-aac5-c68964a4b12e", "content-length": "26", "date": "Thu, 21 Nov 2024 19:43:47 GMT"} })
93252024-11-21T19:43:47.941ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
93262024-11-21T19:43:47.943ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
93272024-11-21T19:43:47.944ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
93282024-11-21T19:43:47.944ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
93292024-11-21T19:43:47.944ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "d3e7012c-7046-4089-8bdc-196a55d51dc7", "content-length": "26", "date": "Thu, 21 Nov 2024 19:43:47 GMT"} })
93302024-11-21T19:43:47.944ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
93312024-11-21T19:43:47.946ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
93322024-11-21T19:43:47.979ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
93332024-11-21T19:43:47.979ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
93342024-11-21T19:43:47.981ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
93352024-11-21T19:43:47.981ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
93362024-11-21T19:43:47.981ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
93372024-11-21T19:43:47.982ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 143.399826ms iteration = 22 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
93382024-11-21T19:43:47.983ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
93392024-11-21T19:43:47.983ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
93402024-11-21T19:43:47.983ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
93412024-11-21T19:43:47.984ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
93422024-11-21T19:43:47.985ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
93432024-11-21T19:43:47.986ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
93442024-11-21T19:43:47.986ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
93452024-11-21T19:43:47.986ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
93462024-11-21T19:43:47.987ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
93472024-11-21T19:43:47.987ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
93482024-11-21T19:43:47.987ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
93492024-11-21T19:43:47.987ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 150.050004ms iteration = 22 status = 0
93502024-11-21T19:43:47.994ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
93512024-11-21T19:43:47.994ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
93522024-11-21T19:43:47.995ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "c68d81f1-0528-4a79-ac27-f3721cd15b6a", "content-length": "26", "date": "Thu, 21 Nov 2024 19:43:47 GMT"} })
93532024-11-21T19:43:47.995ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
93542024-11-21T19:43:47.996ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
93552024-11-21T19:43:48.002ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "c25fbb64-ccbb-4a29-87a9-df01aab86824", "content-length": "372", "date": "Thu, 21 Nov 2024 19:43:48 GMT"} })
93562024-11-21T19:43:48.002ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
93572024-11-21T19:43:48.003ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
93582024-11-21T19:43:48.043ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
93592024-11-21T19:43:48.043ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
93602024-11-21T19:43:48.043ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "6773b78a-b967-4c2f-99f2-b812cb3bf3c6", "date": "Thu, 21 Nov 2024 19:43:48 GMT"} })
93612024-11-21T19:43:48.043ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
93622024-11-21T19:43:48.044ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
93632024-11-21T19:43:48.045ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
93642024-11-21T19:43:48.045ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
93652024-11-21T19:43:48.048ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
93662024-11-21T19:43:48.048ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
93672024-11-21T19:43:48.052ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
93682024-11-21T19:43:48.052ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
93692024-11-21T19:43:48.055ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "990438d5-2e86-441e-b4a7-9c96f2ff09cd", "date": "Thu, 21 Nov 2024 19:43:48 GMT"} })
93702024-11-21T19:43:48.056ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 217.820691ms iteration = 22 status = {}
93712024-11-21T19:43:48.059ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
93722024-11-21T19:43:48.059ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "9d9d5b74-529e-4963-bd6a-c02e2673e61b", "content-length": "1", "date": "Thu, 21 Nov 2024 19:43:47 GMT"} })
93732024-11-21T19:43:48.061ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
93742024-11-21T19:43:48.061ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
93752024-11-21T19:43:48.076ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 238.907612ms iteration = 22 status = {}
93762024-11-21T19:43:48.085ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
93772024-11-21T19:43:48.086ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "218b1285-48a2-416c-b852-553d0fbba57e", "content-length": "691", "date": "Thu, 21 Nov 2024 19:43:48 GMT"} })
93782024-11-21T19:43:48.086ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
93792024-11-21T19:43:48.086ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
93802024-11-21T19:43:48.086ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
93812024-11-21T19:43:48.087ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 249.89313ms iteration = 22 status = null
93822024-11-21T19:43:48.088ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 250.055021ms iteration = 22 status = {}
93832024-11-21T19:43:48.147ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
93842024-11-21T19:43:48.147ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
93852024-11-21T19:43:48.164ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
93862024-11-21T19:43:48.164ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
93872024-11-21T19:43:48.164ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
93882024-11-21T19:43:48.172ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
93892024-11-21T19:43:48.172ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "6fcfc688-5274-4a98-a382-046691b0a914", "content-length": "134", "date": "Thu, 21 Nov 2024 19:43:47 GMT"} })
93902024-11-21T19:43:48.172ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
93912024-11-21T19:43:48.172ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
93922024-11-21T19:43:48.172ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
93932024-11-21T19:43:48.172ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
93942024-11-21T19:43:48.172ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
93952024-11-21T19:43:48.172ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
93962024-11-21T19:43:48.172ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
93972024-11-21T19:43:48.172ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
93982024-11-21T19:43:48.172ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
93992024-11-21T19:43:48.172ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
94002024-11-21T19:43:48.173ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
94012024-11-21T19:43:48.173ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
94022024-11-21T19:43:48.173ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
94032024-11-21T19:43:48.175ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
94042024-11-21T19:43:48.175ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "5b6b5ab7-26f1-43d7-9543-644bf5d3f4e4", "content-length": "2", "date": "Thu, 21 Nov 2024 19:43:47 GMT"} })
94052024-11-21T19:43:48.175ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
94062024-11-21T19:43:48.175ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "fb947a0d-f9db-4c3d-8074-8071fbc4cc7e", "content-length": "51", "date": "Thu, 21 Nov 2024 19:43:47 GMT"} })
94072024-11-21T19:43:48.175ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
94082024-11-21T19:43:48.175ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
94092024-11-21T19:43:48.275ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "55712137-3d0c-40c2-aabc-9eff8fb5b7ab", "date": "Thu, 21 Nov 2024 19:43:48 GMT"} })
94102024-11-21T19:43:48.275ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
94112024-11-21T19:43:48.275ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
94122024-11-21T19:43:48.275ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "28ce3e6b-6d8a-43a3-a261-c5adec39ad6e", "content-length": "682", "date": "Thu, 21 Nov 2024 19:43:48 GMT"} })
94132024-11-21T19:43:48.275ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
94142024-11-21T19:43:48.276ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
94152024-11-21T19:43:48.280ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
94162024-11-21T19:43:48.280ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
94172024-11-21T19:43:48.282ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
94182024-11-21T19:43:48.282ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
94192024-11-21T19:43:48.285ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
94202024-11-21T19:43:48.285ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
94212024-11-21T19:43:48.291ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 453.992086ms iteration = 22 status = {}
94222024-11-21T19:43:56.966ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 302 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = f303a773-0ea4-4a09-9541-6c3a95a20e36 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
94232024-11-21T19:43:57.837ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 65 reason = Timeout
94242024-11-21T19:43:57.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
94252024-11-21T19:43:57.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
94262024-11-21T19:43:57.838ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 65 reason = Timeout
94272024-11-21T19:43:57.838ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
94282024-11-21T19:43:57.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
94292024-11-21T19:43:57.839ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
94302024-11-21T19:43:57.840ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
94312024-11-21T19:43:57.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
94322024-11-21T19:43:57.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
94332024-11-21T19:43:57.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
94342024-11-21T19:43:57.842ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
94352024-11-21T19:43:57.842ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 4.093235ms iteration = 65 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
94362024-11-21T19:43:57.844ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
94372024-11-21T19:43:57.844ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
94382024-11-21T19:43:57.845ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
94392024-11-21T19:43:57.848ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
94402024-11-21T19:43:57.848ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
94412024-11-21T19:43:57.848ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
94422024-11-21T19:43:57.857ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
94432024-11-21T19:43:57.857ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 20.23917ms iteration = 65 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
94442024-11-21T19:44:01.959ZINFOnexus (dropshot_internal): accepted connection file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023 local_addr = [fd00:1122:3344:101::c]:12221 remote_addr = [fd00:1122:3344:101::d]:49223
94452024-11-21T19:44:01.959ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:49223 req_id = ec11dbcd-f2a8-4869-8580-cc6da040c668 roles = RoleSet { roles: {} } uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
94462024-11-21T19:44:01.959ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:49223 req_id = ec11dbcd-f2a8-4869-8580-cc6da040c668 resource = Database result = Ok(()) uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
94472024-11-21T19:44:01.961ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 2125 local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:49223 req_id = ec11dbcd-f2a8-4869-8580-cc6da040c668 response_code = 200 uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?sort_by=id_ascending
94482024-11-21T19:44:01.961ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:49223 req_id = aeda3e4e-80e8-4d1a-a53a-b1574ab64b27 roles = RoleSet { roles: {} } uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
94492024-11-21T19:44:01.961ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:49223 req_id = aeda3e4e-80e8-4d1a-a53a-b1574ab64b27 resource = Database result = Ok(()) uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
94502024-11-21T19:44:01.963ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 2511 local_addr = [fd00:1122:3344:101::c]:12221 method = GET remote_addr = [fd00:1122:3344:101::d]:49223 req_id = aeda3e4e-80e8-4d1a-a53a-b1574ab64b27 response_code = 200 uri = /metrics/collectors/5fb99117-c685-44d5-bec2-c58adb63b619/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJkNWQ5NDczNy00Mzk5LTRjZGQtYjVhNy0xNzZkMzIxYTQyOTgifX0%3D
94512024-11-21T19:44:07.838ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 66 reason = Timeout
94522024-11-21T19:44:07.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
94532024-11-21T19:44:07.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
94542024-11-21T19:44:07.838ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 66 reason = Timeout
94552024-11-21T19:44:07.838ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
94562024-11-21T19:44:07.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
94572024-11-21T19:44:07.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
94582024-11-21T19:44:07.841ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
94592024-11-21T19:44:07.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
94602024-11-21T19:44:07.843ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 4.680633ms iteration = 66 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
94612024-11-21T19:44:07.843ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
94622024-11-21T19:44:07.843ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
94632024-11-21T19:44:07.843ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
94642024-11-21T19:44:07.846ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
94652024-11-21T19:44:07.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
94662024-11-21T19:44:07.847ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
94672024-11-21T19:44:07.850ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
94682024-11-21T19:44:07.850ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
94692024-11-21T19:44:07.850ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
94702024-11-21T19:44:07.859ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
94712024-11-21T19:44:07.859ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 20.95271ms iteration = 66 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
94722024-11-21T19:44:11.966ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 373 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 4b9c84bd-1ccb-45df-971d-83b8a476eb5c response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
94732024-11-21T19:44:17.837ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 23 reason = Timeout
94742024-11-21T19:44:17.837ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
94752024-11-21T19:44:17.837ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 67 reason = Timeout
94762024-11-21T19:44:17.837ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_external dns_group = external iteration = 12 reason = Timeout
94772024-11-21T19:44:17.837ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_external dns_group = external
94782024-11-21T19:44:17.837ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
94792024-11-21T19:44:17.837ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_internal dns_group = internal iteration = 12 reason = Timeout
94802024-11-21T19:44:17.837ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
94812024-11-21T19:44:17.837ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_external dns_group = external iteration = 14 reason = Timeout
94822024-11-21T19:44:17.837ZDEBGnexus (BackgroundTasks): activating background_task = dns_propagation_internal dns_group = internal iteration = 14 reason = Timeout
94832024-11-21T19:44:17.837ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 25 reason = Timeout
94842024-11-21T19:44:17.837ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 23 reason = Timeout
94852024-11-21T19:44:17.837ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
94862024-11-21T19:44:17.837ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 23 reason = Timeout
94872024-11-21T19:44:17.837ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 23 reason = Timeout
94882024-11-21T19:44:17.837ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
94892024-11-21T19:44:17.837ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_executor iteration = 13 reason = Timeout
94902024-11-21T19:44:17.837ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
94912024-11-21T19:44:17.837ZWARNnexus (BackgroundTasks): Blueprint execution: skipped background_task = blueprint_executor file = nexus/src/app/background/tasks/blueprint_execution.rs:83 reason = blueprint disabled target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb
94922024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 23 reason = Timeout
94932024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_executor elapsed = 18.58\u{b5}s iteration = 13 status = {"enabled":false,"target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb"}
94942024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 23 reason = Timeout
94952024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
94962024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activating background_task = decommissioned_disk_cleaner iteration = 12 reason = Timeout
94972024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activating background_task = dns_servers_external dns_group = external iteration = 12 reason = Timeout
94982024-11-21T19:44:17.838ZINFOnexus (BackgroundTasks): Decommissioned disk cleaner running background_task = decommissioned_disk_cleaner file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:56
94992024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
95002024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
95012024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activating background_task = external_endpoints iteration = 12 reason = Timeout
95022024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
95032024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 23 reason = Timeout
95042024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
95052024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activating background_task = metrics_producer_gc iteration = 12 reason = Timeout
95062024-11-21T19:44:17.838ZINFOnexus (BackgroundTasks): Metric producer GC running background_task = metrics_producer_gc expiration = 2024-11-21 19:34:17.838018556 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:48
95072024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activating background_task = dns_config_internal dns_group = internal iteration = 12 reason = Timeout
95082024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): roles background_task = metrics_producer_gc roles = RoleSet { roles: {} }
95092024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): reading DNS version background_task = dns_config_internal dns_group = internal
95102024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
95112024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
95122024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
95132024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
95142024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
95152024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
95162024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
95172024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
95182024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = metrics_producer_gc resource = Database result = Ok(())
95192024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
95202024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
95212024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
95222024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
95232024-11-21T19:44:17.838ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _external-dns._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_external-dns._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("4e669ff4-6dde-4cd2-8491-623460b56d8d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_external-dns._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("90b200e8-d1ee-4e0d-975c-64628c5532d2.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::8))) }], valid_until: Instant { tv_sec: 1400, tv_nsec: 709171119 } })
95242024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
95252024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 23 reason = Timeout
95262024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
95272024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 23 reason = Timeout
95282024-11-21T19:44:17.838ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
95292024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 23 reason = Timeout
95302024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activating background_task = abandoned_vmm_reaper iteration = 12 reason = Timeout
95312024-11-21T19:44:17.838ZDEBGnexus (DnsResolver): lookup_all_socket_v6 srv dns_name = _nameservice._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_nameservice._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0377cf96-baf3-4e09-a570-eb7c67037d4d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("0d1b3b31-4591-4bcb-931d-e8e03365f0d2.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nameservice._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 5353, target: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("112d49fb-486a-4cde-88e1-1221a68bf3b6.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:1::1))) }], valid_until: Instant { tv_sec: 1400, tv_nsec: 709235090 } })
95322024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
95332024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
95342024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): roles background_task = abandoned_vmm_reaper roles = RoleSet { roles: {} }
95352024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activating background_task = lookup_region_port iteration = 12 reason = Timeout
95362024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): roles background_task = lookup_region_port roles = RoleSet { roles: {} }
95372024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activating background_task = instance_reincarnation iteration = 12 reason = Timeout
95382024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
95392024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 23 reason = Timeout
95402024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
95412024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
95422024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 23 reason = Timeout
95432024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
95442024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 23 reason = Timeout
95452024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 23 reason = Timeout
95462024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
95472024-11-21T19:44:17.838ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
95482024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 67 reason = Timeout
95492024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
95502024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
95512024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
95522024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
95532024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
95542024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = lookup_region_port resource = Database result = Ok(())
95552024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = abandoned_vmm_reaper resource = Database result = Ok(())
95562024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } background_task = dns_servers_external current_servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } dns_group = external
95572024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
95582024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_external dns_group = external elapsed = 1.251957ms iteration = 12 status = {"addresses":["[fd00:1122:3344:101::9]:5353","[fd00:1122:3344:101::8]:5353"]}
95592024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
95602024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
95612024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
95622024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
95632024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): found DNS servers (no change) addresses = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } background_task = dns_servers_internal current_servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } dns_group = internal
95642024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_servers_internal dns_group = internal elapsed = 1.522061ms iteration = 12 status = {"addresses":["[fd00:1122:3344:3::1]:5353","[fd00:1122:3344:2::1]:5353","[fd00:1122:3344:1::1]:5353"]}
95652024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
95662024-11-21T19:44:17.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
95672024-11-21T19:44:17.841ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1400, tv_nsec: 712373702 } })
95682024-11-21T19:44:17.842ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
95692024-11-21T19:44:17.842ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
95702024-11-21T19:44:17.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
95712024-11-21T19:44:17.843ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
95722024-11-21T19:44:17.843ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 5.665866ms iteration = 23 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
95732024-11-21T19:44:17.843ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
95742024-11-21T19:44:17.844ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
95752024-11-21T19:44:17.846ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
95762024-11-21T19:44:17.846ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
95772024-11-21T19:44:17.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
95782024-11-21T19:44:17.846ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
95792024-11-21T19:44:17.846ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
95802024-11-21T19:44:17.847ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
95812024-11-21T19:44:17.847ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Fleet result = Ok(())
95822024-11-21T19:44:17.847ZDEBGnexus (BackgroundTasks): roles background_task = decommissioned_disk_cleaner roles = RoleSet { roles: {} }
95832024-11-21T19:44:17.847ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
95842024-11-21T19:44:17.847ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = decommissioned_disk_cleaner resource = Database result = Ok(())
95852024-11-21T19:44:17.847ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
95862024-11-21T19:44:17.848ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
95872024-11-21T19:44:17.848ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
95882024-11-21T19:44:17.848ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
95892024-11-21T19:44:17.848ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
95902024-11-21T19:44:17.849ZINFOnexus (BackgroundTasks): Metric producer GC complete (no errors) background_task = metrics_producer_gc expiration = 2024-11-21 19:34:17.838018556 UTC file = nexus/src/app/background/tasks/metrics_producer_gc.rs:69 pruned = {}
95912024-11-21T19:44:17.849ZDEBGnexus (BackgroundTasks): activation complete background_task = metrics_producer_gc elapsed = 10.990647ms iteration = 12 status = {"expiration":"2024-11-21T19:34:17.838018556Z","pruned":[]}
95922024-11-21T19:44:17.850ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
95932024-11-21T19:44:17.850ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
95942024-11-21T19:44:17.850ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
95952024-11-21T19:44:17.852ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 13.5186ms iteration = 23 status = {"errors":[],"records_set_to_done":[]}
95962024-11-21T19:44:17.852ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
95972024-11-21T19:44:17.852ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
95982024-11-21T19:44:17.852ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
95992024-11-21T19:44:17.852ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1400, tv_nsec: 723120015 } })
96002024-11-21T19:44:17.852ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
96012024-11-21T19:44:17.855ZDEBGnexus (BackgroundTasks): roles background_task = instance_reincarnation roles = RoleSet { roles: {} }
96022024-11-21T19:44:17.855ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_reincarnation resource = Database result = Ok(())
96032024-11-21T19:44:17.858ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
96042024-11-21T19:44:17.858ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
96052024-11-21T19:44:17.860ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
96062024-11-21T19:44:17.860ZDEBGnexus (BackgroundTasks): activation complete background_task = lookup_region_port elapsed = 22.013614ms iteration = 12 status = {"errors":[],"found_port_ok":[]}
96072024-11-21T19:44:17.860ZDEBGnexus (BackgroundTasks): Found abandoned VMMs background_task = abandoned_vmm_reaper count = 0 total = 0
96082024-11-21T19:44:17.860ZINFOnexus (BackgroundTasks): Abandoned VMMs reaped background_task = abandoned_vmm_reaper file = nexus/src/app/background/tasks/abandoned_vmm_reaper.rs:171 sled_reservations_deleted = 0 vmms_already_deleted = 0 vmms_deleted = 0 vmms_found = 0
96092024-11-21T19:44:17.860ZDEBGnexus (BackgroundTasks): activation complete background_task = abandoned_vmm_reaper elapsed = 22.223457ms iteration = 12 status = {"errors":[],"sled_reservations_deleted":0,"vmms_already_deleted":0,"vmms_deleted":0,"vmms_found":0}
96102024-11-21T19:44:17.861ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 22.294988ms iteration = 23 status = {"errors":[],"garbage_collect_requested":[]}
96112024-11-21T19:44:17.862ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
96122024-11-21T19:44:17.862ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
96132024-11-21T19:44:17.862ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
96142024-11-21T19:44:17.862ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
96152024-11-21T19:44:17.862ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
96162024-11-21T19:44:17.864ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
96172024-11-21T19:44:17.865ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
96182024-11-21T19:44:17.865ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
96192024-11-21T19:44:17.865ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
96202024-11-21T19:44:17.865ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
96212024-11-21T19:44:17.865ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
96222024-11-21T19:44:17.866ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
96232024-11-21T19:44:17.866ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
96242024-11-21T19:44:17.866ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
96252024-11-21T19:44:17.868ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
96262024-11-21T19:44:17.868ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
96272024-11-21T19:44:17.868ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
96282024-11-21T19:44:17.869ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1400, tv_nsec: 740067092 } })
96292024-11-21T19:44:17.869ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
96302024-11-21T19:44:17.870ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
96312024-11-21T19:44:17.870ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
96322024-11-21T19:44:17.870ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
96332024-11-21T19:44:17.870ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
96342024-11-21T19:44:17.871ZDEBGnexus (BackgroundTasks): Found zpools on decommissioned disks background_task = decommissioned_disk_cleaner count = 0
96352024-11-21T19:44:17.871ZINFOnexus (BackgroundTasks): Cleaned decommissioned zpools background_task = decommissioned_disk_cleaner deleted = 0 error_count = 0 file = nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs:140 found = 0 not_ready_to_be_deleted = 0
96362024-11-21T19:44:17.871ZDEBGnexus (BackgroundTasks): activation complete background_task = decommissioned_disk_cleaner elapsed = 33.051062ms iteration = 12 status = {"deleted":0,"error":null,"error_count":0,"found":0,"not_ready_to_be_deleted":0}
96372024-11-21T19:44:17.871ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
96382024-11-21T19:44:17.871ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
96392024-11-21T19:44:17.872ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
96402024-11-21T19:44:17.875ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
96412024-11-21T19:44:17.877ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
96422024-11-21T19:44:17.878ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
96432024-11-21T19:44:17.878ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
96442024-11-21T19:44:17.878ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
96452024-11-21T19:44:17.878ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
96462024-11-21T19:44:17.880ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
96472024-11-21T19:44:17.880ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
96482024-11-21T19:44:17.881ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
96492024-11-21T19:44:17.883ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
96502024-11-21T19:44:17.883ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
96512024-11-21T19:44:17.884ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
96522024-11-21T19:44:17.884ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
96532024-11-21T19:44:17.884ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 45.61897ms iteration = 23 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
96542024-11-21T19:44:17.887ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
96552024-11-21T19:44:17.887ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
96562024-11-21T19:44:17.887ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
96572024-11-21T19:44:17.887ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
96582024-11-21T19:44:17.887ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
96592024-11-21T19:44:17.887ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
96602024-11-21T19:44:17.887ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
96612024-11-21T19:44:17.887ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = DnsConfig result = Ok(())
96622024-11-21T19:44:17.887ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
96632024-11-21T19:44:17.888ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
96642024-11-21T19:44:17.888ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
96652024-11-21T19:44:17.890ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
96662024-11-21T19:44:17.890ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
96672024-11-21T19:44:17.890ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 51.489328ms iteration = 23 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
96682024-11-21T19:44:17.890ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
96692024-11-21T19:44:17.893ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
96702024-11-21T19:44:17.893ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
96712024-11-21T19:44:17.893ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
96722024-11-21T19:44:17.893ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
96732024-11-21T19:44:17.893ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
96742024-11-21T19:44:17.895ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
96752024-11-21T19:44:17.895ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
96762024-11-21T19:44:17.897ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Fleet result = Ok(())
96772024-11-21T19:44:17.897ZDEBGnexus (BackgroundTasks): roles background_task = external_endpoints roles = RoleSet { roles: {} }
96782024-11-21T19:44:17.897ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
96792024-11-21T19:44:17.897ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
96802024-11-21T19:44:17.897ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = external_endpoints resource = Database result = Ok(())
96812024-11-21T19:44:17.898ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
96822024-11-21T19:44:17.899ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
96832024-11-21T19:44:17.899ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
96842024-11-21T19:44:17.901ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 62.440274ms iteration = 23 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
96852024-11-21T19:44:17.901ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
96862024-11-21T19:44:17.901ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
96872024-11-21T19:44:17.901ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 63.57988ms iteration = 67 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
96882024-11-21T19:44:17.901ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
96892024-11-21T19:44:17.905ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
96902024-11-21T19:44:17.905ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
96912024-11-21T19:44:17.907ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
96922024-11-21T19:44:17.907ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
96932024-11-21T19:44:17.910ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 71.505736ms iteration = 23 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
96942024-11-21T19:44:17.912ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
96952024-11-21T19:44:17.912ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
96962024-11-21T19:44:17.912ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_external dns_group = external version = 2
96972024-11-21T19:44:17.912ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_external dns_group = external
96982024-11-21T19:44:17.912ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_external dns_group = external
96992024-11-21T19:44:17.912ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
97002024-11-21T19:44:17.913ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
97012024-11-21T19:44:17.914ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
97022024-11-21T19:44:17.914ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
97032024-11-21T19:44:17.914ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 76.652354ms iteration = 23 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
97042024-11-21T19:44:17.915ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
97052024-11-21T19:44:17.915ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
97062024-11-21T19:44:17.917ZDEBGnexus (BackgroundTasks): found DNS version background_task = dns_config_internal dns_group = internal version = 1
97072024-11-21T19:44:17.917ZDEBGnexus (BackgroundTasks): reading DNS config background_task = dns_config_internal dns_group = internal
97082024-11-21T19:44:17.917ZDEBGnexus (BackgroundTasks): reading DNS zones background_task = dns_config_internal dns_group = internal
97092024-11-21T19:44:17.917ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
97102024-11-21T19:44:17.917ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
97112024-11-21T19:44:17.918ZWARNnexus (BackgroundTasks): instance reincarnation completed with errors background_task = instance_reincarnation file = nexus/src/app/background/tasks/instance_reincarnation.rs:103 instances_changed_state = 0 instances_found = 0 instances_reincarnated = 0 query_errors = 0 restart_errors = 0
97122024-11-21T19:44:17.918ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_reincarnation elapsed = 80.014019ms iteration = 12 status = {"changed_state":[],"disabled":false,"errors":[],"instances_found":{"Failed":0,"SagaUnwound":0},"instances_reincarnated":[],"restart_errors":[]}
97132024-11-21T19:44:17.919ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
97142024-11-21T19:44:17.919ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
97152024-11-21T19:44:17.919ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
97162024-11-21T19:44:17.920ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
97172024-11-21T19:44:17.920ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
97182024-11-21T19:44:17.920ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
97192024-11-21T19:44:17.922ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
97202024-11-21T19:44:17.922ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_external count = 1 dns_group = external
97212024-11-21T19:44:17.922ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_external batch_size = 100 dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 0 version = 2
97222024-11-21T19:44:17.922ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
97232024-11-21T19:44:17.922ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
97242024-11-21T19:44:17.922ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 84.962495ms iteration = 23 status = 0
97252024-11-21T19:44:17.925ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
97262024-11-21T19:44:17.925ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
97272024-11-21T19:44:17.925ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
97282024-11-21T19:44:17.926ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
97292024-11-21T19:44:17.926ZDEBGnexus (BackgroundTasks): found zones background_task = dns_config_internal count = 1 dns_group = internal
97302024-11-21T19:44:17.926ZDEBGnexus (BackgroundTasks): listing DNS names for zone background_task = dns_config_internal batch_size = 100 dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 0 version = 1
97312024-11-21T19:44:17.926ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
97322024-11-21T19:44:17.926ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
97332024-11-21T19:44:17.929ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = DnsConfig result = Ok(())
97342024-11-21T19:44:17.929ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_external dns_group = external roles = RoleSet { roles: {} }
97352024-11-21T19:44:17.929ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_external dns_group = external resource = Database result = Ok(())
97362024-11-21T19:44:17.931ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
97372024-11-21T19:44:17.931ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_external dns_group = external dns_zone_id = 54cb9a9c-588d-41b0-9b8c-aa84dfcc59b0 dns_zone_name = oxide.test found_so_far = 1 version = 2
97382024-11-21T19:44:17.931ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_external dns_group = external nzones = 1 version = 2
97392024-11-21T19:44:17.931ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_external current_generation = 2 current_time_created = 2024-11-21 19:33:32.758954 UTC dns_group = external generation = 2
97402024-11-21T19:44:17.931ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_external dns_group = external elapsed = 93.985366ms iteration = 12 status = {"generation":2}
97412024-11-21T19:44:17.933ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = DnsConfig result = Ok(())
97422024-11-21T19:44:17.933ZDEBGnexus (BackgroundTasks): roles background_task = dns_config_internal dns_group = internal roles = RoleSet { roles: {} }
97432024-11-21T19:44:17.933ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = dns_config_internal dns_group = internal resource = Database result = Ok(())
97442024-11-21T19:44:17.935ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 97.383551ms iteration = 25 status = {"physical_disks_added":0}
97452024-11-21T19:44:17.935ZDEBGnexus (BackgroundTasks): found all DNS names for zone background_task = dns_config_internal dns_group = internal dns_zone_id = 3d5fcde1-45b9-4eef-b092-78b429ed7718 dns_zone_name = control-plane.oxide.internal found_so_far = 51 version = 1
97462024-11-21T19:44:17.935ZDEBGnexus (BackgroundTasks): read DNS config background_task = dns_config_internal dns_group = internal nzones = 1 version = 1
97472024-11-21T19:44:17.935ZDEBGnexus (BackgroundTasks): found latest DNS generation (unchanged) background_task = dns_config_internal current_generation = 1 current_time_created = 2024-11-21 19:33:31.797390 UTC dns_group = internal generation = 1
97482024-11-21T19:44:17.935ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_config_internal dns_group = internal elapsed = 97.937159ms iteration = 12 status = {"generation":1}
97492024-11-21T19:44:17.941ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 103.101667ms iteration = 67 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
97502024-11-21T19:44:17.948ZDEBGnexus (BackgroundTasks): found Silo/DNS/TLS config (no change) background_task = external_endpoints config = ExternalEndpoints { by_dns_name: {"default-silo.sys.oxide.test": ExternalEndpoint { silo_id: 001de000-5110-4000-8000-000000000000, db_silo: Silo { identity: SiloIdentity { id: 001de000-5110-4000-8000-000000000000, name: Name(Name("default-silo")), description: "default silo", time_created: 2024-11-21T19:33:15.446964Z, time_modified: 2024-11-21T19:33:15.446964Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {}, rcgen: Generation(Generation(1)) }, tls_certs: [] }, "recovery.sys.oxide.test": ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }}, warnings: [NoSiloCerts { silo_id: 001de000-5110-4000-8000-000000000000, dns_name: "default-silo.sys.oxide.test" }], default_endpoint: Some(ExternalEndpoint { silo_id: 4d2571c7-98ea-428d-998e-b26e88b12e96, db_silo: Silo { identity: SiloIdentity { id: 4d2571c7-98ea-428d-998e-b26e88b12e96, name: Name(Name("recovery")), description: "built-in recovery Silo", time_created: 2024-11-21T19:33:32.672523Z, time_modified: 2024-11-21T19:33:32.672523Z, time_deleted: None }, discoverable: false, authentication_mode: Local, user_provision_type: ApiOnly, mapped_fleet_roles: Object {"admin": Array [String("admin")]}, rcgen: Generation(Generation(3)) }, tls_certs: [TlsCertificate { digest: "dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0" }] }) }
97512024-11-21T19:44:17.948ZDEBGnexus (BackgroundTasks): activation complete background_task = external_endpoints elapsed = 110.161402ms iteration = 12 status = {"by_dns_name":{"default-silo.sys.oxide.test":{"silo_id":"001de000-5110-4000-8000-000000000000","tls_certs":[]},"recovery.sys.oxide.test":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]}},"default_endpoint":{"silo_id":"4d2571c7-98ea-428d-998e-b26e88b12e96","tls_certs":["dbd466c2b03229a969fa7c94302bfd3670651d44733517f4c1cd0c29c5dd0aa0"]},"warnings":["silo 001de000-5110-4000-8000-000000000000 with DNS name \\"default-silo.sys.oxide.test\\" has no usable certificates"]}
97522024-11-21T19:44:18.042ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
97532024-11-21T19:44:18.042ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
97542024-11-21T19:44:18.042ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "6e3a6a94-e67c-4b92-ad1c-37dfecc79c65", "content-length": "26", "date": "Thu, 21 Nov 2024 19:44:17 GMT"} })
97552024-11-21T19:44:18.042ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
97562024-11-21T19:44:18.044ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
97572024-11-21T19:44:18.139ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::9]:5353/config
97582024-11-21T19:44:18.158ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:3::1]:5353/config
97592024-11-21T19:44:18.163ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
97602024-11-21T19:44:18.163ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
97612024-11-21T19:44:18.178ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
97622024-11-21T19:44:18.178ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
97632024-11-21T19:44:18.179ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "4fbf5c95-303f-48ee-9c77-e077040a0d8c", "content-length": "26", "date": "Thu, 21 Nov 2024 19:44:17 GMT"} })
97642024-11-21T19:44:18.179ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
97652024-11-21T19:44:18.180ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
97662024-11-21T19:44:18.185ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
97672024-11-21T19:44:18.221ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
97682024-11-21T19:44:18.221ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
97692024-11-21T19:44:18.221ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "f9321998-f91c-43c1-9a86-920fc0d94ba6", "content-length": "26", "date": "Thu, 21 Nov 2024 19:44:17 GMT"} })
97702024-11-21T19:44:18.221ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
97712024-11-21T19:44:18.222ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "5380ea6b-6230-43de-8f21-326a377ca506", "content-length": "372", "date": "Thu, 21 Nov 2024 19:44:18 GMT"} })
97722024-11-21T19:44:18.222ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
97732024-11-21T19:44:18.222ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "d06e4b96-7b6b-4c51-a1b0-d5690be576a7", "content-length": "691", "date": "Thu, 21 Nov 2024 19:44:18 GMT"} })
97742024-11-21T19:44:18.222ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
97752024-11-21T19:44:18.222ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
97762024-11-21T19:44:18.223ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
97772024-11-21T19:44:18.223ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
97782024-11-21T19:44:18.224ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
97792024-11-21T19:44:18.225ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 387.511788ms iteration = 23 status = {}
97802024-11-21T19:44:18.237ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
97812024-11-21T19:44:18.237ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
97822024-11-21T19:44:18.237ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "6de62d45-4779-43f1-a1a5-6fd2c04a8d9c", "date": "Thu, 21 Nov 2024 19:44:18 GMT"} })
97832024-11-21T19:44:18.237ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
97842024-11-21T19:44:18.238ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
97852024-11-21T19:44:18.238ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
97862024-11-21T19:44:18.238ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "28e82cf4-c30c-4367-862b-550f5c5b5a54", "content-length": "1", "date": "Thu, 21 Nov 2024 19:44:18 GMT"} })
97872024-11-21T19:44:18.240ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
97882024-11-21T19:44:18.240ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
97892024-11-21T19:44:18.240ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
97902024-11-21T19:44:18.240ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
97912024-11-21T19:44:18.241ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 403.668963ms iteration = 23 status = null
97922024-11-21T19:44:18.244ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
97932024-11-21T19:44:18.244ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
97942024-11-21T19:44:18.247ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
97952024-11-21T19:44:18.247ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
97962024-11-21T19:44:18.247ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "2a2d4d5c-4464-432c-83d8-ed6b7db1b005", "date": "Thu, 21 Nov 2024 19:44:18 GMT"} })
97972024-11-21T19:44:18.249ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 411.132364ms iteration = 23 status = {}
97982024-11-21T19:44:18.294ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_external body = Some(Body) dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] } uri = http://[fd00:1122:3344:101::8]:5353/config
97992024-11-21T19:44:18.294ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::8]:5353/config", status: 204, headers: {"x-request-id": "c66e1718-fafa-419d-9aa9-a56d10a6fd68", "date": "Thu, 21 Nov 2024 19:44:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
98002024-11-21T19:44:18.294ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::8]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
98012024-11-21T19:44:18.297ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 generation = 2 result = Ok(Response { url: "http://[fd00:1122:3344:101::9]:5353/config", status: 204, headers: {"x-request-id": "5ff63848-77fe-4cee-a4fe-7278ec7dbb9e", "date": "Thu, 21 Nov 2024 19:44:17 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
98022024-11-21T19:44:18.297ZINFOnexus (BackgroundTasks): DNS server now at generation 2 background_task = dns_propagation_external dns_group = external dns_server_url = http://[fd00:1122:3344:101::9]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
98032024-11-21T19:44:18.298ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_external dns_group = external file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 2 servers = DnsServersList { addresses: [[fd00:1122:3344:101::9]:5353, [fd00:1122:3344:101::8]:5353] }
98042024-11-21T19:44:18.298ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_external dns_group = external elapsed = 460.813397ms iteration = 14 status = {"generation":2,"server_results":{"[fd00:1122:3344:101::8]:5353":{"Ok":null},"[fd00:1122:3344:101::9]:5353":{"Ok":null}}}
98052024-11-21T19:44:18.304ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 467.051411ms iteration = 23 status = {}
98062024-11-21T19:44:18.309ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:2::1]:5353/config
98072024-11-21T19:44:18.373ZDEBGnexus (BackgroundTasks): client request background_task = dns_propagation_internal body = Some(Body) dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 method = PUT servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] } uri = http://[fd00:1122:3344:1::1]:5353/config
98082024-11-21T19:44:18.374ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:2::1]:5353/config", status: 204, headers: {"x-request-id": "9d1aa426-f61b-4863-b550-48fa6b042e65", "date": "Thu, 21 Nov 2024 19:44:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
98092024-11-21T19:44:18.374ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:2::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
98102024-11-21T19:44:18.376ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:1::1]:5353/config", status: 204, headers: {"x-request-id": "45e4cea4-648e-43fa-9f8d-17d959a75a49", "date": "Thu, 21 Nov 2024 19:44:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
98112024-11-21T19:44:18.376ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:1::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
98122024-11-21T19:44:18.378ZDEBGnexus (BackgroundTasks): client response background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 generation = 1 result = Ok(Response { url: "http://[fd00:1122:3344:3::1]:5353/config", status: 204, headers: {"x-request-id": "9cbaa846-6e20-4fc5-91db-474442f26e9f", "date": "Thu, 21 Nov 2024 19:44:18 GMT"} }) servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
98132024-11-21T19:44:18.378ZINFOnexus (BackgroundTasks): DNS server now at generation 1 background_task = dns_propagation_internal dns_group = internal dns_server_url = http://[fd00:1122:3344:3::1]:5353 file = nexus/src/app/background/tasks/dns_propagation.rs:169 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
98142024-11-21T19:44:18.380ZINFOnexus (BackgroundTasks): DNS propagation: done background_task = dns_propagation_internal dns_group = internal file = nexus/src/app/background/tasks/dns_propagation.rs:106 generation = 1 servers = DnsServersList { addresses: [[fd00:1122:3344:3::1]:5353, [fd00:1122:3344:2::1]:5353, [fd00:1122:3344:1::1]:5353] }
98152024-11-21T19:44:18.380ZDEBGnexus (BackgroundTasks): activation complete background_task = dns_propagation_internal dns_group = internal elapsed = 542.257215ms iteration = 14 status = {"generation":1,"server_results":{"[fd00:1122:3344:1::1]:5353":{"Ok":null},"[fd00:1122:3344:2::1]:5353":{"Ok":null},"[fd00:1122:3344:3::1]:5353":{"Ok":null}}}
98162024-11-21T19:44:18.407ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
98172024-11-21T19:44:18.407ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
98182024-11-21T19:44:18.409ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
98192024-11-21T19:44:18.409ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
98202024-11-21T19:44:18.410ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
98212024-11-21T19:44:18.419ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
98222024-11-21T19:44:18.419ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "5c1e35e2-4d83-453e-899f-8f9eb5e6b71c", "content-length": "134", "date": "Thu, 21 Nov 2024 19:44:18 GMT"} })
98232024-11-21T19:44:18.419ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
98242024-11-21T19:44:18.419ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
98252024-11-21T19:44:18.420ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
98262024-11-21T19:44:18.420ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
98272024-11-21T19:44:18.420ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
98282024-11-21T19:44:18.420ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
98292024-11-21T19:44:18.420ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
98302024-11-21T19:44:18.420ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
98312024-11-21T19:44:18.420ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
98322024-11-21T19:44:18.420ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
98332024-11-21T19:44:18.420ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
98342024-11-21T19:44:18.420ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
98352024-11-21T19:44:18.420ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
98362024-11-21T19:44:18.423ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
98372024-11-21T19:44:18.423ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "ea71535a-f529-4bf3-9189-c3d097b4c844", "content-length": "2", "date": "Thu, 21 Nov 2024 19:44:18 GMT"} })
98382024-11-21T19:44:18.423ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
98392024-11-21T19:44:18.423ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "49d7b45f-96fa-48d4-9e34-8c3586042991", "content-length": "51", "date": "Thu, 21 Nov 2024 19:44:18 GMT"} })
98402024-11-21T19:44:18.423ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
98412024-11-21T19:44:18.423ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
98422024-11-21T19:44:18.523ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "0b9716b7-cb21-4d83-9fea-dd448703a991", "date": "Thu, 21 Nov 2024 19:44:18 GMT"} })
98432024-11-21T19:44:18.523ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
98442024-11-21T19:44:18.523ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
98452024-11-21T19:44:18.524ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "625e7889-5259-40a1-bcb2-3bc304aeb9e0", "content-length": "682", "date": "Thu, 21 Nov 2024 19:44:18 GMT"} })
98462024-11-21T19:44:18.524ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
98472024-11-21T19:44:18.524ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
98482024-11-21T19:44:18.527ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
98492024-11-21T19:44:18.528ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
98502024-11-21T19:44:18.531ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
98512024-11-21T19:44:18.531ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
98522024-11-21T19:44:18.533ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
98532024-11-21T19:44:18.533ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
98542024-11-21T19:44:18.540ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 702.595698ms iteration = 23 status = {}
98552024-11-21T19:44:18.617ZDEBGnexus (producer-registration-task): registering / renewing oximeter producer lease with Nexus
98562024-11-21T19:44:18.617ZDEBGnexus (producer-registration-task): using nexus address for registration addr = [fd00:1122:3344:101::c]:12221
98572024-11-21T19:44:18.647ZDEBGnexus (producer-registration-task): client request body = Some(Body) method = POST uri = http://[fd00:1122:3344:101::c]:12221/metrics/producers
98582024-11-21T19:44:18.647ZINFOnexus (dropshot_internal): accepted connection file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023 local_addr = [fd00:1122:3344:101::c]:12221 remote_addr = [fd00:1122:3344:101::c]:61829
98592024-11-21T19:44:18.647ZDEBGnexus (dropshot_internal): roles actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:61829 req_id = 579d1810-7202-401c-8914-86f56ea147c6 roles = RoleSet { roles: {} } uri = /metrics/producers
98602024-11-21T19:44:18.648ZDEBGnexus (dropshot_internal): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) actor_id = 001de000-05e4-4000-8000-000000000002 authenticated = true local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:61829 req_id = 579d1810-7202-401c-8914-86f56ea147c6 resource = Database result = Ok(()) uri = /metrics/producers
98612024-11-21T19:44:18.687ZINFOnexus: assigned collector to new producer collector_id = 5fb99117-c685-44d5-bec2-c58adb63b619 file = nexus/src/app/oximeter.rs:76 producer_id = 915e11b8-d7cb-4730-9906-7980540813dc
98622024-11-21T19:44:18.687ZINFOnexus (dropshot_internal): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 39369 local_addr = [fd00:1122:3344:101::c]:12221 method = POST remote_addr = [fd00:1122:3344:101::c]:61829 req_id = 579d1810-7202-401c-8914-86f56ea147c6 response_code = 201 uri = /metrics/producers
98632024-11-21T19:44:18.687ZDEBGnexus (producer-registration-task): client response result = Ok(Response { url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", status: 201, headers: {"content-type": "application/json", "x-request-id": "579d1810-7202-401c-8914-86f56ea147c6", "content-length": "41", "date": "Thu, 21 Nov 2024 19:44:18 GMT"} })
98642024-11-21T19:44:18.689ZDEBGnexus (producer-registration-task): registered with nexus successfully
98652024-11-21T19:44:18.689ZDEBGnexus (producer-registration-task): pausing until time to renew lease lease_duration = 600s wait_period = 150s
98662024-11-21T19:44:26.966ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 453 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = bde0db28-6ef4-4a3e-8185-6e17686603ac response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
98672024-11-21T19:44:27.838ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 68 reason = Timeout
98682024-11-21T19:44:27.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
98692024-11-21T19:44:27.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
98702024-11-21T19:44:27.839ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 68 reason = Timeout
98712024-11-21T19:44:27.839ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
98722024-11-21T19:44:27.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
98732024-11-21T19:44:27.841ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
98742024-11-21T19:44:27.843ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
98752024-11-21T19:44:27.843ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
98762024-11-21T19:44:27.844ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
98772024-11-21T19:44:27.845ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
98782024-11-21T19:44:27.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
98792024-11-21T19:44:27.847ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
98802024-11-21T19:44:27.850ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
98812024-11-21T19:44:27.850ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
98822024-11-21T19:44:27.850ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
98832024-11-21T19:44:27.865ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
98842024-11-21T19:44:27.865ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 26.577025ms iteration = 68 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
98852024-11-21T19:44:27.874ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
98862024-11-21T19:44:27.874ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
98872024-11-21T19:44:27.876ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 36.68489ms iteration = 68 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
98882024-11-21T19:44:37.838ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 69 reason = Timeout
98892024-11-21T19:44:37.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
98902024-11-21T19:44:37.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
98912024-11-21T19:44:37.839ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 69 reason = Timeout
98922024-11-21T19:44:37.839ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
98932024-11-21T19:44:37.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
98942024-11-21T19:44:37.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
98952024-11-21T19:44:37.841ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
98962024-11-21T19:44:37.841ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
98972024-11-21T19:44:37.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
98982024-11-21T19:44:37.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
98992024-11-21T19:44:37.843ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
99002024-11-21T19:44:37.843ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 4.47838ms iteration = 69 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
99012024-11-21T19:44:37.845ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
99022024-11-21T19:44:37.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
99032024-11-21T19:44:37.847ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
99042024-11-21T19:44:37.849ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
99052024-11-21T19:44:37.849ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
99062024-11-21T19:44:37.849ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
99072024-11-21T19:44:37.862ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
99082024-11-21T19:44:37.862ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 24.613539ms iteration = 69 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
99092024-11-21T19:44:41.967ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 284 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = 2c88f9ca-6c5b-47ad-a0b1-f2ecd6f0b373 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
99102024-11-21T19:44:47.838ZDEBGnexus (BackgroundTasks): activating background_task = bfd_manager iteration = 24 reason = Timeout
99112024-11-21T19:44:47.838ZDEBGnexus (BackgroundTasks): roles background_task = bfd_manager roles = RoleSet { roles: {} }
99122024-11-21T19:44:47.838ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 70 reason = Timeout
99132024-11-21T19:44:47.838ZDEBGnexus (BackgroundTasks): activating background_task = nat_v4_garbage_collector iteration = 24 reason = Timeout
99142024-11-21T19:44:47.838ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
99152024-11-21T19:44:47.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
99162024-11-21T19:44:47.838ZDEBGnexus (BackgroundTasks): activating background_task = physical_disk_adoption iteration = 26 reason = Timeout
99172024-11-21T19:44:47.838ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement iteration = 24 reason = Timeout
99182024-11-21T19:44:47.838ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
99192024-11-21T19:44:47.838ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
99202024-11-21T19:44:47.838ZDEBGnexus (BackgroundTasks): activating background_task = v2p_manager iteration = 24 reason = Timeout
99212024-11-21T19:44:47.838ZDEBGnexus (BackgroundTasks): activating background_task = service_zone_nat_tracker iteration = 24 reason = Timeout
99222024-11-21T19:44:47.838ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
99232024-11-21T19:44:47.838ZDEBGnexus (BackgroundTasks): activating background_task = switch_port_config_manager iteration = 24 reason = Timeout
99242024-11-21T19:44:47.838ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
99252024-11-21T19:44:47.838ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
99262024-11-21T19:44:47.838ZDEBGnexus (BackgroundTasks): activating background_task = phantom_disks iteration = 24 reason = Timeout
99272024-11-21T19:44:47.838ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = bfd_manager resource = Database result = Ok(())
99282024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
99292024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
99302024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
99312024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
99322024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
99332024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
99342024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
99352024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 70 reason = Timeout
99362024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_garbage_collection iteration = 24 reason = Timeout
99372024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_garbage_collection roles = RoleSet { roles: {} }
99382024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_start iteration = 24 reason = Timeout
99392024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
99402024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
99412024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): activating background_task = vpc_route_manager iteration = 24 reason = Timeout
99422024-11-21T19:44:47.839ZINFOnexus (BackgroundTasks): VPC route manager running background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:60
99432024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
99442024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_finish iteration = 24 reason = Timeout
99452024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_finish roles = RoleSet { roles: {} }
99462024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): activating background_task = instance_updater iteration = 24 reason = Timeout
99472024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): looking for instances with destroyed active VMMs... background_task = instance_updater
99482024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
99492024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): activating background_task = region_snapshot_replacement_step iteration = 24 reason = Timeout
99502024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): activating background_task = instance_watcher iteration = 24 reason = Timeout
99512024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
99522024-11-21T19:44:47.839ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
99532024-11-21T19:44:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_garbage_collection resource = Database result = Ok(())
99542024-11-21T19:44:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
99552024-11-21T19:44:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
99562024-11-21T19:44:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_finish resource = Database result = Ok(())
99572024-11-21T19:44:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
99582024-11-21T19:44:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
99592024-11-21T19:44:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
99602024-11-21T19:44:47.840ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
99612024-11-21T19:44:47.841ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1430, tv_nsec: 711870575 } })
99622024-11-21T19:44:47.842ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = bfd_manager file = nexus/src/app/mod.rs:1131
99632024-11-21T19:44:47.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
99642024-11-21T19:44:47.842ZDEBGnexus (BackgroundTasks): activation complete background_task = phantom_disks elapsed = 3.924892ms iteration = 24 status = {"phantom_disk_deleted_err":0,"phantom_disk_deleted_ok":0}
99652024-11-21T19:44:47.842ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1430, tv_nsec: 712797297 } })
99662024-11-21T19:44:47.842ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
99672024-11-21T19:44:47.842ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1131
99682024-11-21T19:44:47.843ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
99692024-11-21T19:44:47.845ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
99702024-11-21T19:44:47.845ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
99712024-11-21T19:44:47.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
99722024-11-21T19:44:47.846ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
99732024-11-21T19:44:47.846ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
99742024-11-21T19:44:47.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
99752024-11-21T19:44:47.846ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Fleet result = Ok(())
99762024-11-21T19:44:47.846ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
99772024-11-21T19:44:47.846ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
99782024-11-21T19:44:47.848ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
99792024-11-21T19:44:47.848ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_garbage_collection elapsed = 8.704676ms iteration = 24 status = {"errors":[],"garbage_collect_requested":[]}
99802024-11-21T19:44:47.849ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_finish elapsed = 10.047854ms iteration = 24 status = {"errors":[],"records_set_to_done":[]}
99812024-11-21T19:44:47.850ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Inventory result = Ok(())
99822024-11-21T19:44:47.850ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
99832024-11-21T19:44:47.851ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
99842024-11-21T19:44:47.853ZINFOnexus (BackgroundTasks): listed instances with destroyed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
99852024-11-21T19:44:47.853ZDEBGnexus (BackgroundTasks): looking for instances with failed active VMMs... background_task = instance_updater
99862024-11-21T19:44:47.853ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
99872024-11-21T19:44:47.854ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
99882024-11-21T19:44:47.855ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
99892024-11-21T19:44:47.856ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
99902024-11-21T19:44:47.856ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
99912024-11-21T19:44:47.857ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
99922024-11-21T19:44:47.862ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
99932024-11-21T19:44:47.863ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
99942024-11-21T19:44:47.864ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
99952024-11-21T19:44:47.864ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
99962024-11-21T19:44:47.865ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
99972024-11-21T19:44:47.865ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
99982024-11-21T19:44:47.865ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
99992024-11-21T19:44:47.866ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Fleet result = Ok(())
100002024-11-21T19:44:47.866ZDEBGnexus (BackgroundTasks): roles background_task = instance_watcher roles = RoleSet { roles: {} }
100012024-11-21T19:44:47.866ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_watcher resource = Database result = Ok(())
100022024-11-21T19:44:47.868ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
100032024-11-21T19:44:47.868ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
100042024-11-21T19:44:47.869ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
100052024-11-21T19:44:47.869ZINFOnexus (BackgroundTasks): listed instances with failed active VMMs background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
100062024-11-21T19:44:47.869ZDEBGnexus (BackgroundTasks): looking for instances with terminated active migrations... background_task = instance_updater
100072024-11-21T19:44:47.869ZDEBGnexus (BackgroundTasks): roles background_task = instance_updater roles = RoleSet { roles: {} }
100082024-11-21T19:44:47.870ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = instance_updater resource = Database result = Ok(())
100092024-11-21T19:44:47.872ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
100102024-11-21T19:44:47.872ZDEBGnexus (DnsResolver): lookup_all_ipv6 srv dns_name = _dendrite._tcp.control-plane.oxide.internal response = SrvLookup(Lookup { query: Query { name: Name("_dendrite._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_dendrite._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12224, target: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("dendrite-203795ed-576b-40e4-bed8-7463e3f2162c.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::2))) }], valid_until: Instant { tv_sec: 1430, tv_nsec: 742992681 } })
100112024-11-21T19:44:47.873ZINFOnexus (BackgroundTasks): Determining switch slots managed by switch zones background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1131 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
100122024-11-21T19:44:47.874ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
100132024-11-21T19:44:47.874ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
100142024-11-21T19:44:47.874ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
100152024-11-21T19:44:47.874ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Fleet result = Ok(())
100162024-11-21T19:44:47.874ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
100172024-11-21T19:44:47.874ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
100182024-11-21T19:44:47.876ZINFOnexus (BackgroundTasks): all instance checks complete background_task = instance_watcher file = nexus/src/app/background/tasks/instance_watcher.rs:555 pruned_instances = 0 total_completed = 0 total_failed = 0 total_incomplete = 0 total_instances = 0 update_sagas_queued = 0
100192024-11-21T19:44:47.876ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_watcher elapsed = 37.153707ms iteration = 24 status = {"failed_checks":{},"incomplete_checks":{},"instance_states":{},"pruned_instances":0,"total_instances":0,"update_sagas_queued":0}
100202024-11-21T19:44:47.889ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
100212024-11-21T19:44:47.889ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 51.041862ms iteration = 70 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}
100222024-11-21T19:44:47.891ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement roles = RoleSet { roles: {} }
100232024-11-21T19:44:47.891ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement resource = Database result = Ok(())
100242024-11-21T19:44:47.896ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
100252024-11-21T19:44:47.897ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
100262024-11-21T19:44:47.898ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_start roles = RoleSet { roles: {} }
100272024-11-21T19:44:47.898ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_start resource = Database result = Ok(())
100282024-11-21T19:44:47.898ZDEBGnexus (BackgroundTasks): roles background_task = region_snapshot_replacement_step roles = RoleSet { roles: {} }
100292024-11-21T19:44:47.899ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_snapshot_replacement_step resource = Database result = Ok(())
100302024-11-21T19:44:47.901ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
100312024-11-21T19:44:47.901ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
100322024-11-21T19:44:47.910ZINFOnexus (BackgroundTasks): listed instances with terminated active migrations background_task = instance_updater count = 0 file = nexus/src/app/background/tasks/instance_updater.rs:59
100332024-11-21T19:44:47.910ZINFOnexus (BackgroundTasks): instance updater activation completed background_task = instance_updater destroyed_active_vmms = 0 failed_active_vmms = 0 file = nexus/src/app/background/tasks/instance_updater.rs:225 terminated_active_migrations = 0 update_sagas_completed = 0 update_sagas_started = 0
100342024-11-21T19:44:47.910ZDEBGnexus (BackgroundTasks): activation complete background_task = instance_updater elapsed = 70.35867ms iteration = 24 status = {"destroyed_active_vmms":0,"disabled":false,"failed_active_vmms":0,"query_errors":[],"saga_errors":[],"sagas_completed":0,"sagas_started":0,"terminated_active_migrations":0}
100352024-11-21T19:44:47.914ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
100362024-11-21T19:44:47.917ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 77.543136ms iteration = 70 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
100372024-11-21T19:44:47.917ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Fleet result = Ok(())
100382024-11-21T19:44:47.917ZDEBGnexus (BackgroundTasks): roles background_task = physical_disk_adoption roles = RoleSet { roles: {} }
100392024-11-21T19:44:47.918ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = physical_disk_adoption resource = Database result = Ok(())
100402024-11-21T19:44:47.921ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement elapsed = 83.318343ms iteration = 24 status = {"errors":[],"requests_completed_ok":[],"requests_created_ok":[],"start_invoked_ok":[]}
100412024-11-21T19:44:47.928ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_start elapsed = 88.620254ms iteration = 24 status = {"errors":[],"requests_created_ok":[],"start_invoked_ok":[]}
100422024-11-21T19:44:47.929ZDEBGnexus (BackgroundTasks): activation complete background_task = region_snapshot_replacement_step elapsed = 89.720759ms iteration = 24 status = {"errors":[],"step_garbage_collect_invoked_ok":[],"step_invoked_ok":[],"step_records_created_ok":[]}
100432024-11-21T19:44:47.931ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
100442024-11-21T19:44:47.931ZDEBGnexus (MgsClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
100452024-11-21T19:44:47.931ZDEBGnexus (MgsClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "7d629e3d-1c09-44a0-b9ea-642c714f68ad", "content-length": "26", "date": "Thu, 21 Nov 2024 19:44:47 GMT"} })
100462024-11-21T19:44:47.931ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
100472024-11-21T19:44:47.932ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = nat_v4_garbage_collector file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
100482024-11-21T19:44:47.943ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1139 zone_address = fd00:1122:3344:101::2
100492024-11-21T19:44:47.943ZDEBGnexus (MgsClient): client request background_task = bfd_manager body = None method = GET uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
100502024-11-21T19:44:47.944ZDEBGnexus (MgsClient): client response background_task = bfd_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "555bd1e7-493c-477e-87ce-ce505b3b280a", "content-length": "26", "date": "Thu, 21 Nov 2024 19:44:47 GMT"} })
100512024-11-21T19:44:47.944ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = bfd_manager file = nexus/src/app/mod.rs:1144 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
100522024-11-21T19:44:47.946ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = bfd_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n}
100532024-11-21T19:44:47.955ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
100542024-11-21T19:44:47.955ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
100552024-11-21T19:44:47.984ZINFOnexus (BackgroundTasks): determining switch slot managed by dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1139 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 zone_address = fd00:1122:3344:101::2
100562024-11-21T19:44:47.984ZDEBGnexus (MgsClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12225/local/switch-id
100572024-11-21T19:44:47.984ZDEBGnexus (MgsClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12225/local/switch-id", status: 200, headers: {"content-type": "application/json", "x-request-id": "775f0c05-7592-4959-bd0d-a705aa94bb55", "content-length": "26", "date": "Thu, 21 Nov 2024 19:44:47 GMT"} })
100582024-11-21T19:44:47.984ZINFOnexus (BackgroundTasks): identified switch slot for dendrite zone background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1144 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 slot = SpIdentifier {\n slot: 0,\n type_: Switch,\n} zone_address = fd00:1122:3344:101::2
100592024-11-21T19:44:47.986ZINFOnexus (BackgroundTasks): completed mapping dendrite zones to switch slots background_task = switch_port_config_manager file = nexus/src/app/mod.rs:1176 mappings = {\n Switch0: fd00:1122:3344:101::2,\n} rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
100602024-11-21T19:44:47.990ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:98
100612024-11-21T19:44:47.990ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
100622024-11-21T19:44:47.993ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
100632024-11-21T19:44:47.993ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
100642024-11-21T19:44:47.995ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
100652024-11-21T19:44:47.997ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Sled { parent: Fleet, key: 203795ed-576b-40e4-bed8-7463e3f2162c, lookup_type: ById(203795ed-576b-40e4-bed8-7463e3f2162c) } result = Ok(())
100662024-11-21T19:44:47.997ZDEBGnexus (BackgroundTasks): roles background_task = service_zone_nat_tracker roles = RoleSet { roles: {} }
100672024-11-21T19:44:47.998ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = service_zone_nat_tracker resource = Database result = Ok(())
100682024-11-21T19:44:47.999ZDEBGnexus (BackgroundTasks): activation complete background_task = service_zone_nat_tracker elapsed = 160.794909ms iteration = 24 status = 0
100692024-11-21T19:44:48.025ZDEBGnexus (DpdClient): client request background_task = nat_v4_garbage_collector body = None method = GET uri = http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen
100702024-11-21T19:44:48.026ZDEBGnexus (DpdClient): client response background_task = nat_v4_garbage_collector result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/rpw/nat/ipv4/gen", status: 200, headers: {"content-type": "application/json", "x-request-id": "b94853a5-41b9-43de-852e-443a64a6181b", "content-length": "1", "date": "Thu, 21 Nov 2024 19:44:47 GMT"} })
100712024-11-21T19:44:48.027ZDEBGnexus (BackgroundTasks): activation complete background_task = physical_disk_adoption elapsed = 188.473908ms iteration = 26 status = {"physical_disks_added":0}
100722024-11-21T19:44:48.027ZDEBGnexus (BackgroundTasks): roles background_task = nat_v4_garbage_collector roles = RoleSet { roles: {} }
100732024-11-21T19:44:48.027ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = nat_v4_garbage_collector resource = Database result = Ok(())
100742024-11-21T19:44:48.029ZDEBGnexus (BackgroundTasks): activation complete background_task = nat_v4_garbage_collector elapsed = 190.472485ms iteration = 24 status = null
100752024-11-21T19:44:48.029ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 200, headers: {"content-type": "application/json", "x-request-id": "fa280388-cc4a-4ed8-83d5-b517f7a02b88", "content-length": "372", "date": "Thu, 21 Nov 2024 19:44:48 GMT"} })
100762024-11-21T19:44:48.029ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
100772024-11-21T19:44:48.030ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
100782024-11-21T19:44:48.034ZDEBGnexus (BackgroundTasks): activation complete background_task = bfd_manager elapsed = 195.631194ms iteration = 24 status = {}
100792024-11-21T19:44:48.070ZINFOnexus (BackgroundTasks): computed internet gateway mappings for sled assocs = {} background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:116 sled = buskin
100802024-11-21T19:44:48.070ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/eip-gateways
100812024-11-21T19:44:48.071ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/eip-gateways", status: 204, headers: {"x-request-id": "e889e3d5-6bfc-476b-8cec-2d5250d3c431", "date": "Thu, 21 Nov 2024 19:44:47 GMT"} })
100822024-11-21T19:44:48.071ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
100832024-11-21T19:44:48.071ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
100842024-11-21T19:44:48.073ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
100852024-11-21T19:44:48.073ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
100862024-11-21T19:44:48.077ZDEBGnexus (BackgroundTasks): roles background_task = vpc_route_manager roles = RoleSet { roles: {} }
100872024-11-21T19:44:48.077ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = vpc_route_manager resource = Database result = Ok(())
100882024-11-21T19:44:48.081ZINFOnexus (BackgroundTasks): VPC route manager sled 203795ed-576b-40e4-bed8-7463e3f2162c push not needed background_task = vpc_route_manager file = nexus/src/app/background/tasks/vpc_routes.rs:262
100892024-11-21T19:44:48.081ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager body = Some(Body) method = PUT uri = http://[fd00:1122:3344:101::1]:12345/vpc-routes
100902024-11-21T19:44:48.087ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
100912024-11-21T19:44:48.087ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
100922024-11-21T19:44:48.088ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
100932024-11-21T19:44:48.090ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
100942024-11-21T19:44:48.090ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
100952024-11-21T19:44:48.090ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
100962024-11-21T19:44:48.092ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
100972024-11-21T19:44:48.093ZDEBGnexus (BackgroundTasks): authorize result action = ListChildren actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Fleet result = Ok(())
100982024-11-21T19:44:48.093ZDEBGnexus (BackgroundTasks): roles background_task = v2p_manager roles = RoleSet { roles: {} }
100992024-11-21T19:44:48.093ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = v2p_manager resource = Database result = Ok(())
101002024-11-21T19:44:48.108ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
101012024-11-21T19:44:48.108ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
101022024-11-21T19:44:48.110ZINFOnexus (BackgroundTasks): fetching switch port settings background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:151 port = SwitchPort { id: 58e95733-2442-4d09-af49-856b85bc788c, rack_id: 6f50eaaf-6509-4592-a570-21594f7b8789, switch_location: "switch0", port_name: "qsfp0", port_settings_id: Some(a0508820-29d4-4ad9-a25a-a54581875a4f) } rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
101032024-11-21T19:44:48.110ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
101042024-11-21T19:44:48.110ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
101052024-11-21T19:44:48.119ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron
101062024-11-21T19:44:48.120ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/port/qsfp0/settings?tag=omicron", status: 200, headers: {"content-type": "application/json", "x-request-id": "5d8570de-b9bd-4003-a59f-70421afb76a5", "content-length": "134", "date": "Thu, 21 Nov 2024 19:44:47 GMT"} })
101072024-11-21T19:44:48.120ZINFOnexus (BackgroundTasks): retrieved port settings from switch background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1746 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 settings = PortSettings { links: {"0": LinkSettings { addrs: [10.151.1.174], params: LinkCreate { autoneg: false, fec: Some(None), kr: false, lane: Some(LinkId(0)), speed: Speed40G, tx_eq: None } }} } switch_port_id = "qsfp0"
101082024-11-21T19:44:48.120ZINFOnexus (BackgroundTasks): port settings up to date, skipping background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1773 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0 switch_port_id = "qsfp0" switch_port_settings_id = a0508820-29d4-4ad9-a25a-a54581875a4f
101092024-11-21T19:44:48.120ZINFOnexus (BackgroundTasks): retrieved existing routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:383 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, Some(1))}}
101102024-11-21T19:44:48.120ZINFOnexus (BackgroundTasks): retrieved desired routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:387 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: {(10.151.1.100, Prefix4 { length: 0, value: 0.0.0.0 }, None, None)}}
101112024-11-21T19:44:48.120ZINFOnexus (BackgroundTasks): calculated static routes to add background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:396 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
101122024-11-21T19:44:48.120ZINFOnexus (BackgroundTasks): calculated static routes to delete background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:402 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
101132024-11-21T19:44:48.120ZINFOnexus (BackgroundTasks): deleting static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:407 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
101142024-11-21T19:44:48.120ZINFOnexus (BackgroundTasks): removing static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1920 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = DeleteStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
101152024-11-21T19:44:48.120ZINFOnexus (BackgroundTasks): adding static routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:413 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 routes = {Switch0: AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } }}
101162024-11-21T19:44:48.120ZINFOnexus (BackgroundTasks): adding static v4 routes background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:1956 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 request = AddStaticRoute4Request { routes: StaticRoute4List { list: [StaticRoute4 { nexthop: 10.151.1.100, prefix: Prefix4 { length: 0, value: 0.0.0.0 }, rib_priority: 1, vlan_id: None }] } } switch_location = Switch0
101172024-11-21T19:44:48.120ZINFOnexus (BackgroundTasks): checking for changes to loopback addresses background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:422 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789
101182024-11-21T19:44:48.120ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
101192024-11-21T19:44:48.121ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
101202024-11-21T19:44:48.122ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv4
101212024-11-21T19:44:48.122ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv4", status: 200, headers: {"content-type": "application/json", "x-request-id": "583b8239-9b91-4aad-8dea-9e5329b53687", "content-length": "2", "date": "Thu, 21 Nov 2024 19:44:47 GMT"} })
101222024-11-21T19:44:48.122ZDEBGnexus (DpdClient): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::2]:12224/loopback/ipv6
101232024-11-21T19:44:48.122ZDEBGnexus (DpdClient): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::2]:12224/loopback/ipv6", status: 200, headers: {"content-type": "application/json", "x-request-id": "a93ea880-e190-4fc9-9cc4-15e1919da9aa", "content-length": "51", "date": "Thu, 21 Nov 2024 19:44:47 GMT"} })
101242024-11-21T19:44:48.122ZINFOnexus (BackgroundTasks): applying SMF config uplink updates to switch zone background_task = switch_port_config_manager config = [HostPortConfig { addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], lldp: Some(LldpPortConfig { chassis_id: None, management_addrs: None, port_description: None, port_id: None, status: Disabled, system_description: None, system_name: None }), port: "qsfp0", tx_eq: None }] file = nexus/src/app/background/tasks/sync_switch_configuration.rs:471 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch_location = Switch0
101252024-11-21T19:44:48.122ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = Some(Body) method = POST rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/switch-ports
101262024-11-21T19:44:48.185ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = vpc_route_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/vpc-routes", status: 204, headers: {"x-request-id": "c3de0d9d-a658-44e0-8f60-b799814731e5", "date": "Thu, 21 Nov 2024 19:44:47 GMT"} })
101272024-11-21T19:44:48.186ZDEBGnexus (BackgroundTasks): activation complete background_task = vpc_route_manager elapsed = 346.987456ms iteration = 24 status = {}
101282024-11-21T19:44:48.187ZDEBGnexus (BackgroundTasks): client request SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager body = None method = GET uri = http://[fd00:1122:3344:101::1]:12345/v2p
101292024-11-21T19:44:48.187ZDEBGnexus (BackgroundTasks): client response SledAgent = 203795ed-576b-40e4-bed8-7463e3f2162c background_task = v2p_manager result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/v2p", status: 200, headers: {"content-type": "application/json", "x-request-id": "ca7efe22-1d43-414e-96b1-200a0518a975", "content-length": "691", "date": "Thu, 21 Nov 2024 19:44:48 GMT"} })
101302024-11-21T19:44:48.187ZINFOnexus (BackgroundTasks): found opte v2p mappings background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:102 interfaces = {} sled = buskin
101312024-11-21T19:44:48.187ZINFOnexus (BackgroundTasks): v2p mappings to delete background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:114 mappings = [] sled = buskin
101322024-11-21T19:44:48.187ZINFOnexus (BackgroundTasks): v2p mappings to add background_task = v2p_manager file = nexus/src/app/background/tasks/v2p_mappings.rs:127 mappings = [] sled = buskin
101332024-11-21T19:44:48.188ZDEBGnexus (BackgroundTasks): activation complete background_task = v2p_manager elapsed = 350.2959ms iteration = 24 status = {}
101342024-11-21T19:44:48.279ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/switch-ports", status: 204, headers: {"x-request-id": "e84e3c12-ec30-41ae-b349-488bfc3a1ee2", "date": "Thu, 21 Nov 2024 19:44:48 GMT"} })
101352024-11-21T19:44:48.279ZINFOnexus (BackgroundTasks): no bgp config found for switch, skipping. background_task = switch_port_config_manager file = nexus/src/app/background/tasks/sync_switch_configuration.rs:774 rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 switch = Switch0
101362024-11-21T19:44:48.279ZDEBGnexus (BackgroundTasks): client request background_task = switch_port_config_manager body = None method = GET rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 uri = http://[fd00:1122:3344:101::1]:12345/network-bootstore-config
101372024-11-21T19:44:48.279ZDEBGnexus (BackgroundTasks): client response background_task = switch_port_config_manager rack_id = 6f50eaaf-6509-4592-a570-21594f7b8789 result = Ok(Response { url: "http://[fd00:1122:3344:101::1]:12345/network-bootstore-config", status: 200, headers: {"content-type": "application/json", "x-request-id": "e5e9eaf0-5e42-4f7d-b272-118d2578aea5", "content-length": "682", "date": "Thu, 21 Nov 2024 19:44:48 GMT"} })
101382024-11-21T19:44:48.279ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
101392024-11-21T19:44:48.280ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
101402024-11-21T19:44:48.283ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
101412024-11-21T19:44:48.284ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
101422024-11-21T19:44:48.287ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
101432024-11-21T19:44:48.287ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
101442024-11-21T19:44:48.289ZDEBGnexus (BackgroundTasks): roles background_task = switch_port_config_manager roles = RoleSet { roles: {} }
101452024-11-21T19:44:48.289ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = switch_port_config_manager resource = Database result = Ok(())
101462024-11-21T19:44:48.296ZDEBGnexus (BackgroundTasks): activation complete background_task = switch_port_config_manager elapsed = 458.097111ms iteration = 24 status = {}
101472024-11-21T19:44:56.966ZINFOnexus (dropshot): request completed file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863 latency_us = 334 local_addr = [fd00:1122:3344:101::c]:46619 method = GET remote_addr = [fd00:1122:3344:101::d]:56113 req_id = c8461bfe-7be2-4c5b-97b8-1e9edb5eda00 response_code = 200 uri = /915e11b8-d7cb-4730-9906-7980540813dc
101482024-11-21T19:44:57.838ZDEBGnexus (BackgroundTasks): activating background_task = blueprint_loader iteration = 71 reason = Timeout
101492024-11-21T19:44:57.838ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
101502024-11-21T19:44:57.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
101512024-11-21T19:44:57.839ZDEBGnexus (BackgroundTasks): activating background_task = region_replacement_driver iteration = 71 reason = Timeout
101522024-11-21T19:44:57.839ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
101532024-11-21T19:44:57.839ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
101542024-11-21T19:44:57.840ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
101552024-11-21T19:44:57.842ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = BlueprintConfig result = Ok(())
101562024-11-21T19:44:57.842ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
101572024-11-21T19:44:57.843ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
101582024-11-21T19:44:57.844ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
101592024-11-21T19:44:57.845ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
101602024-11-21T19:44:57.868ZDEBGnexus (BackgroundTasks): roles background_task = region_replacement_driver roles = RoleSet { roles: {} }
101612024-11-21T19:44:57.868ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = region_replacement_driver resource = Database result = Ok(())
101622024-11-21T19:44:57.870ZDEBGnexus (BackgroundTasks): activation complete background_task = region_replacement_driver elapsed = 31.133116ms iteration = 71 status = {"drive_invoked_ok":[],"errors":[],"finish_invoked_ok":[]}
101632024-11-21T19:44:57.874ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {(Fleet, 001de000-1334-4000-8000-000000000000, "admin")} }
101642024-11-21T19:44:57.876ZDEBGnexus (BackgroundTasks): authorize result action = Read actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Blueprint { parent: Fleet, key: 50bfe1c4-af2b-4319-bf0c-306a5806cfeb, lookup_type: ById(50bfe1c4-af2b-4319-bf0c-306a5806cfeb) } result = Ok(())
101652024-11-21T19:44:57.876ZDEBGnexus (BackgroundTasks): roles background_task = blueprint_loader roles = RoleSet { roles: {} }
101662024-11-21T19:44:57.877ZDEBGnexus (BackgroundTasks): authorize result action = Query actor = Some(Actor::UserBuiltin { user_builtin_id: 001de000-05e4-4000-8000-000000000002, .. }) background_task = blueprint_loader resource = Database result = Ok(())
101672024-11-21T19:44:57.888ZDEBGnexus (BackgroundTasks): found latest target blueprint (unchanged) background_task = blueprint_loader original_target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb original_time_created = 2024-11-21 19:33:13.736634 UTC target_id = 50bfe1c4-af2b-4319-bf0c-306a5806cfeb time_created = 2024-11-21 19:33:13.736634 UTC
101682024-11-21T19:44:57.888ZDEBGnexus (BackgroundTasks): activation complete background_task = blueprint_loader elapsed = 49.986257ms iteration = 71 status = {"enabled":false,"status":"target blueprint unchanged","target_id":"50bfe1c4-af2b-4319-bf0c-306a5806cfeb","time_created":"2024-11-21T19:33:13.736634Z"}